Hello Navi

# -- coding: utf-8 --
#!/usr/bin/env python3
"""HackThisSite Programming 4 manual-reading assistant.

Modes:
  offline: read a saved XML;
  live:    GET a fresh XML into scratch using HTS_COOKIE.

The script performs only fetch/decompress/parse/render. It never POSTs and never
writes the cookie. It creates five complete colour layers in answer order:
01-blue.png, 02-green.png, 03-red.png, 04-yellow.png, 05-white.png.
Each image keeps the original 1000x1000 canvas and all geometry for that colour.
"""
from __future__ import annotations

import argparse
import bz2
import hashlib
import json
import math
import os
import re
import sys
import time
import xml.etree.ElementTree as ET
from pathlib import Path
from urllib.request import Request, urlopen

from PIL import Image, ImageDraw

BASE = "https://www.hackthissite.org"
UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
      "Chrome/131.0.0.0 Safari/537.36")
ORDER = ("blue", "green", "red", "yellow", "white")
RGB = {
    "blue": (0, 0, 255),
    "green": (0, 255, 0),
    "red": (255, 0, 0),
    "yellow": (255, 255, 0),
    "white": (255, 255, 255),
}
CANVAS = 1000
SCALE = 4
WIDTH = 2 * SCALE


def fetch_url(url: str, headers: dict[str, str]):
    request = Request(url, headers=headers)
    with urlopen(request, timeout=20) as response:
        return response.read(), dict(response.headers)


def fetch_live(cookie: str, out: Path):
    """Fetch and decompress one instance; cookie is kept only in memory."""
    out.mkdir(parents=True, exist_ok=True)
    headers = {
        "User-Agent": UA,
        "Cookie": cookie,
        "Referer": BASE + "/missions/programming/",
    }
    page_bytes, _ = fetch_url(BASE + "/missions/prog/4/", headers)
    page_text = page_bytes.decode("utf-8", errors="replace")
    links = re.findall(
        r'href=["\']([^"\']*prog/4/(?:XML|xml)[^"\']*)',
        page_text,
        flags=re.I,
    )
    url = (
        BASE + links[0] if links and links[0].startswith("/")
        else links[0] if links else BASE + "/missions/prog/4/XML/"
    )
    if not url.endswith("/"):
        url += "/"
    packed, response_headers = fetch_url(url, headers)
    if packed[:3] != b"BZh":
        raise RuntimeError(
            "XML endpoint did not return bzip2; "
            f"bytes={len(packed)} content_type={response_headers.get('Content-Type')}"
        )
    raw = bz2.decompress(packed)
    xml = out / "instance.xml"
    (out / "instance.xml.bz2").write_bytes(packed)
    xml.write_bytes(raw)
    return xml, url, packed, raw


def local(tag):
    return tag.rsplit("}", 1)[-1]


def load_xml(path: Path):
    raw = path.read_bytes()
    if raw.startswith(b"BZh"):
        raw = bz2.decompress(raw)
    return ET.fromstring(raw)


def draw_colour(xml: Path, colour: str, dst: Path):
    """Render one complete colour layer on the original 1000x1000 canvas."""
    root = load_xml(xml)
    image = Image.new("RGB", (CANVAS * SCALE, CANVAS * SCALE), "black")
    draw = ImageDraw.Draw(image)
    count = 0
    for element in root:
        kind = local(element.tag)
        data = {local(x.tag): (x.text or "").strip() for x in element}
        current = (data.get("Color") or "white").lower()
        if current != colour:
            continue
        count += 1
        ink = RGB[colour]
        if kind == "Line":
            x0 = float(data["XStart"]) * SCALE
            y0 = (CANVAS - float(data["YStart"])) * SCALE
            x1 = float(data["XEnd"]) * SCALE
            y1 = (CANVAS - float(data["YEnd"])) * SCALE
            draw.line((x0, y0, x1, y1), fill=ink, width=WIDTH)
        elif kind == "Arc":
            cx = float(data["XCenter"])
            cy = float(data["YCenter"])
            radius = float(data["Radius"])
            start = float(data["ArcStart"])
            extend = float(data["ArcExtend"])
            box = (
                (cx - radius) * SCALE,
                (CANVAS - (cy + radius)) * SCALE,
                (cx + radius) * SCALE,
                (CANVAS - (cy - radius)) * SCALE,
            )
            draw.arc(box, -(start + extend), -start, fill=ink, width=WIDTH)
    image.resize((CANVAS, CANVAS), Image.Resampling.LANCZOS).save(dst)
    return count


def render(xml: Path, out: Path):
    out.mkdir(parents=True, exist_ok=True)
    counts = {}
    files = {}
    for index, colour in enumerate(ORDER, 1):
        dst = out / f"{index:02d}-{colour}.png"
        counts[colour] = draw_colour(xml, colour, dst)
        files[colour] = str(dst)
    return counts, files


def main():
    ap = argparse.ArgumentParser()
    src = ap.add_mutually_exclusive_group(required=True)
    src.add_argument("--xml", type=Path, help="use a saved XML; no network")
    src.add_argument("--bz2", type=Path, help="use a saved bzip2 XML; decompress locally")
    src.add_argument("--live", action="store_true", help="fetch one fresh XML")
    ap.add_argument("--cookie", default=os.environ.get("HTS_COOKIE"),
                    help="session cookie for --live; never written")
    ap.add_argument("--out", type=Path, default=None,
                    help="output directory; defaults to Hermes scratch")
    args = ap.parse_args()
    started = time.monotonic()
    if args.live:
        if not args.cookie:
            raise SystemExit("--live requires HTS_COOKIE or --cookie")
        out = args.out or Path("hts4-manual-" + str(int(time.time())))
        xml, url, packed, raw = fetch_live(args.cookie, out)
    else:
        out = args.out or Path("hts4-manual-offline")
        out.mkdir(parents=True, exist_ok=True)
        if args.bz2:
            packed = args.bz2.read_bytes()
            if packed[:3] != b"BZh":
                raise SystemExit(f"not a bzip2 file: {args.bz2}")
            raw = bz2.decompress(packed)
            xml = out / "instance.xml"
            xml.write_bytes(raw)
            url = "offline-bz2"
        else:
            xml = args.xml.resolve()
            url, packed, raw = "offline", b"", xml.read_bytes()
    counts, files = render(xml, out / "colours")
    result = {
        "xml": str(xml),
        "url": url,
        "xml_sha256": hashlib.sha256(raw).hexdigest(),
        "bz2_sha256": hashlib.sha256(packed).hexdigest() if packed else None,
        "out": str(out),
        "colour_dir": str(out / "colours"),
        "files": files,
        "counts": counts,
        "elapsed_s": round(time.monotonic() - started, 3),
        "posted": False,
    }
    (out / "result.json").write_text(
        json.dumps(result, indent=2), encoding="utf-8"
    )
    print(json.dumps(result, indent=2))
    print("Read montages in order: " + ",".join(ORDER))


if __name__ == "__main__":
    main()

data = open("6578747261637400.jpg", "rb").read()
eoi = data.rfind(b"\xff\xd9")
print("file len :", len(data))
print("EOI at   :", eoi)
print("trailing :", len(data) - eoi - 2, "bytes")
print("signature:", data[eoi + 2:eoi + 9])

from PIL import Image

im = Image.open("key.jpg").convert("L")
w, h = im.size
for y in range(h):
    print("".join("#" if im.getpixel((x, y)) < 128 else "." for x in range(w)))

from PIL import Image
import numpy as np

a = np.asarray(Image.open("6578747261637400.jpg").convert("L")).astype(int)
b = np.clip((255 - a) * 255.0 / (a.max() - a.min()), 0, 255).astype("uint8")
Image.fromarray(b).resize((a.shape[1] * 8, a.shape[0] * 8)).save("cipher_zoom.png")

def affine_decrypt(cipher, a=5, b=10, alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
    a_inv = pow(a, -1, len(alphabet))
    out = []
    for ch in cipher:
        if ch.upper() in alphabet:
            x = alphabet.index(ch.upper())
            out.append(alphabet[(a_inv * (x - b)) % len(alphabet)])
        else:
            out.append(ch)
    return "".join(out)

def affine_encrypt(plain, a=5, b=10, alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
    out = []
    for ch in plain:
        if ch.upper() in alphabet:
            x = alphabet.index(ch.upper())
            out.append(alphabet[(a * x + b) % len(alphabet)])
        else:
            out.append(ch)
    return "".join(out)

ct = "PGNNZCFYXD"
pt = affine_decrypt(ct)
print(pt)
print(affine_encrypt(pt) == ct)  # True：往返一致

from pathlib import Path

raw = Path("13.bmp").read_bytes()
junk = b"I thought what I'd do was, I'd  pretend I was one of those deaf-mutes"

positions = []
start = 0
while True:
    pos = raw.find(junk, start)
    if pos < 0:
        break
    positions.append(pos)
    start = pos + 1

print("file length:", len(raw))
print("junk length:", len(junk))
print("junk occurrences:", len(positions))
print("junk bytes:", len(junk) * len(positions))

#!/usr/bin/env python3
import struct
from pathlib import Path

source = Path("13.bmp").read_bytes()
junk = b"I thought what I'd do was, I'd  pretend I was one of those deaf-mutes"

clean = bytearray()
pos = 0
junk_count = 0
zero_count = 0

while pos < len(source):
    if source.startswith(junk, pos):
        junk_count += 1
        pos += len(junk)
        while pos < len(source) and source[pos] == 0:
            zero_count += 1
            pos += 1
        continue

    clean.append(source[pos])
    pos += 1

clean = bytes(clean)
Path("13_clean.bmp").write_bytes(clean)

_, declared_size, _, _, pixel_offset = struct.unpack_from(
    "<2sIHHI", clean, 0
)
_, width, height, _, bpp, _, image_size = struct.unpack_from(
    "<IiiHHII", clean, 14
)
row_bytes = ((width * (bpp // 8) + 3) // 4) * 4
expected_size = pixel_offset + row_bytes * abs(height)

print("junk blocks:", junk_count)
print("zero padding:", zero_count)
print("clean length:", len(clean))
print("declared size:", declared_size)
print("expected size:", expected_size)
print("image size:", image_size)
assert len(clean) == declared_size == expected_size

import struct

d = open("12.bmp", "rb").read()
bf_type, bf_size, _, _, off = struct.unpack_from("<2sIHHI", d, 0)
hsz, w, h, planes, bpp, comp, img_size = struct.unpack_from("<IiiHHII", d, 14)
row = ((w * bpp + 31) // 32) * 4                 # 8bpp 每行按 4 字节对齐
print("bfType", bf_type, "bfSize", bf_size, "dataOffset", off)
print("w", w, "h", h, "bpp", bpp, "biSizeImage", img_size)
print("row_bytes", row, "pixel_bytes", row * h)
print("palette_entries", (off - 14 - hsz) // 4)
print("trailing", len(d) - off - row * h)

import struct

d = open("12.bmp", "rb").read()
off = struct.unpack_from("<I", d, 10)[0]
pal = d[14 + 40:off]
gray = all(tuple(pal[i:i + 3]) == (i // 4, i // 4, i // 4) for i in range(0, len(pal), 4))
print("palette is grayscale identity:", gray)

import io
import struct
import zipfile

d = open("12.bmp", "rb").read()
off = struct.unpack_from("<I", d, 10)[0]
w, h, planes, bpp = struct.unpack_from("<iiHH", d, 18)
row = ((w * bpp + 31) // 32) * 4
px = d[off:off + row * h]
decoded = bytes(255 - b for b in px)              # bitwise NOT
with zipfile.ZipFile(io.BytesIO(decoded)) as z:
    print(z.namelist())
    print(z.read("pass.txt").decode())

import struct

d = open("ecstasy-atb-music.wav", "rb").read()
print("riff", d[:4], "file_size", len(d))
pos = 12
while pos + 8 <= len(d):
    cid = d[pos:pos + 4].decode("ascii", "replace")
    size = struct.unpack_from("<I", d, pos + 4)[0]
    print("%-4s offset=%d size=%d" % (cid, pos + 8, size))
    pos += 8 + size + (size & 1)          # 块按 2 字节对齐
print("end_of_chunks", pos, "== file size", pos == len(d))

import wave
import numpy as np

with wave.open("ecstasy-atb-music.wav", "rb") as w:
    sr = w.getframerate()
    pcm = np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float64)
L, R = pcm[0::2], pcm[1::2]
print("corr(L,R) = %.6f" % np.corrcoef(L, R)[0, 1])
print("std L=%.1f R=%.1f  L-R=%.1f  L+R=%.1f" % (
    L.std(), R.std(), (L - R).std(), (L + R).std()))

import wave
import numpy as np

with wave.open("ecstasy-atb-music.wav", "rb") as w:
    sr, nch, sw = w.getframerate(), w.getnchannels(), w.getsampwidth()
    pcm = np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float64)
residual = (pcm[0::2] + pcm[1::2]).astype(np.int16)
with wave.open("residual_sum.wav", "wb") as o:
    o.setnchannels(1)
    o.setsampwidth(sw)
    o.setframerate(sr)
    o.writeframes(residual.tobytes())
print("residual std=%.1f absmax=%d" % (residual.std(), np.abs(residual).max()))

import wave
import numpy as np

with wave.open("ecstasy-atb-music.wav", "rb") as w:
    sr = w.getframerate()
    pcm = np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float64)
res = pcm[0::2] + pcm[1::2]

chunk = res[100000:100000 + 8192] * np.hanning(8192)
spec = np.abs(np.fft.rfft(chunk))
freqs = np.fft.rfftfreq(8192, 1.0 / sr)
band = (freqs > 500) & (freqs < 2000)
top = band.nonzero()[0][np.argsort(spec[band])[-5:]][::-1]
for i in top:
    print("%7.1f Hz  mag %8.1f" % (freqs[i], spec[i]))

import wave
import numpy as np
from scipy.signal import butter, hilbert, sosfiltfilt

with wave.open("ecstasy-atb-music.wav", "rb") as w:
    sr = w.getframerate()
    pcm = np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float64)
res = pcm[0::2] + pcm[1::2]

sos = butter(4, [800 / (sr / 2), 1200 / (sr / 2)], btype="band", output="sos")
env = np.abs(hilbert(sosfiltfilt(sos, res)))
env = np.convolve(env, np.ones(32) / 32, mode="same")
step = sr // 10
print(" ".join("%4d" % env[i * step:(i + 1) * step].mean()
               for i in range(30)))

import wave
import numpy as np
from scipy.signal import butter, hilbert, sosfiltfilt

with wave.open("ecstasy-atb-music.wav", "rb") as w:
    sr = w.getframerate()
    pcm = np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float64)
res = pcm[0::2] + pcm[1::2]

sos = butter(4, [800 / (sr / 2), 1200 / (sr / 2)], btype="band", output="sos")
env = np.abs(hilbert(sosfiltfilt(sos, res)))
env = np.convolve(env, np.ones(32) / 32, mode="same")
on = env > env.max() * 0.25
first, last = np.flatnonzero(on)[[0, -1]]
on = on[first:last + 1]

runs = []
cur, start = bool(on[0]), 0
for i in range(1, len(on)):
    if bool(on[i]) != cur:
        runs.append((cur, i - start))
        cur, start = bool(on[i]), i
runs.append((cur, len(on) - start))

ms = lambda n: round(n / sr * 1000)
print("on  ms:", [ms(n) for c, n in runs if c][:24])
print("off ms:", [ms(n) for c, n in runs if not c][:24])

import wave
import numpy as np
from scipy.signal import butter, hilbert, sosfiltfilt

MORSE = {
    "-----": "0", ".----": "1", "..---": "2", "...--": "3", "....-": "4",
    ".....": "5", "-....": "6", "--...": "7", "---..": "8", "----.": "9",
}

with wave.open("ecstasy-atb-music.wav", "rb") as w:
    sr = w.getframerate()
    pcm = np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float64)
res = pcm[0::2] + pcm[1::2]

sos = butter(4, [800 / (sr / 2), 1200 / (sr / 2)], btype="band", output="sos")
env = np.abs(hilbert(sosfiltfilt(sos, res)))
env = np.convolve(env, np.ones(32) / 32, mode="same")
on = env > env.max() * 0.25
first, last = np.flatnonzero(on)[[0, -1]]
on = on[first:last + 1]

runs = []
cur, start = bool(on[0]), 0
for i in range(1, len(on)):
    if bool(on[i]) != cur:
        runs.append((cur, i - start))
        cur, start = bool(on[i]), i
runs.append((cur, len(on) - start))

unit = min(n for c, n in runs if c)         # 最短的通段 = 一个点
words, word, sym = [], [], ""
for is_on, n in runs:
    if is_on:
        sym += "." if n < unit * 2.5 else "-"
    elif n < unit * 4:                      # 字符内间隔，忽略
        continue
    elif n < unit * 7:                      # 字符间间隔
        word.append(sym)
        sym = ""
    else:                                   # 词间间隔
        word.append(sym)
        words.append(word)
        word, sym = [], ""
if sym:
    word.append(sym)
if word:
    words.append(word)

digits = ["".join(MORSE[s] for s in w) for w in words]
print("morse digits:", " ".join(digits))
print("ascii       :", " ".join(chr(int(d)) for d in digits))

import struct

d = open("stego8.bmp", "rb").read()
bf_type, bf_size, _, _, off = struct.unpack_from("<2sIHHI", d, 0)
hsz, w, h, _, bpp, comp, img_size = struct.unpack_from("<IiiHHII", d, 14)
row = ((w * 3 + 3) // 4) * 4                 # 24bpp 每行按 4 字节对齐
print("bfType", bf_type, "bfSize", bf_size, "dataOffset", off)
print("w", w, "h", h, "bpp", bpp, "biSizeImage", img_size)
print("row_bytes", row, "pixel_bytes", row * h)
print("trailing", len(d) - off - row * h)

import numpy as np
from PIL import Image

a = np.array(Image.open("stego8.bmp").convert("RGB"))
for i, name in enumerate("RGB"):
    vals, counts = np.unique(a[:, :, i], return_counts=True)
    top = sorted(zip(counts.tolist(), vals.tolist()), reverse=True)[:4]
    print(name, "uniq =", len(vals), "top =", top)

import numpy as np
from PIL import Image

a = np.array(Image.open("stego8.bmp").convert("RGB")).astype(int)
r, g, b = a[:, :, 0], a[:, :, 1], a[:, :, 2]
print("R!=G:", int((r != g).sum()), "G!=B:", int((g != b).sum()))
ys, xs = np.where(r != g)
print("bbox x=[%d..%d] y=[%d..%d]" % (xs.min(), xs.max(), ys.min(), ys.max()))

import numpy as np
from PIL import Image

a = np.array(Image.open("stego8.bmp").convert("RGB")).astype(int)
for x in range(66, 85):
    print(x, tuple(a[50, x].tolist()))

import numpy as np
from PIL import Image

a = np.array(Image.open("stego8.bmp").convert("RGB")).astype(int)
r, g, b = a[:, :, 0], a[:, :, 1], a[:, :, 2]
ys, xs = np.where((r != g) & (g == b))            # 红独有像素 = 载体
order = np.argsort(xs)
xs, ys = xs[order], ys[order]
text = "".join(chr(int(a[y, x, 0])) for x, y in zip(xs.tolist(), ys.tolist()))
print(text)
print(text.split("=", 1)[1])                      # 取等号后的答案

#!/usr/bin/env python3
"""HackThisSite Steganography 6 - extract the appended payload from stego6.png.

The PNG container is normal (IHDR / bKGD / tIME / IDAT / IEND).  After the IEND
chunk there are 92 extra bytes that are not part of the image stream: an ASCII
Base64 string.  Decoding it yields a sentence that carries the password.
"""
import base64
import sys


def extract(path):
    data = open(path, "rb").read()
    i = data.find(b"IEND")               # IEND is unique in a valid PNG
    end = i + 8                          # chunk length(4) + type(4) = end of IEND
    trailer = data[end:].strip()         # everything after the PNG stream
    message = base64.b64decode(trailer).decode("ascii")
    return end, trailer.decode("ascii"), message


def main():
    path = sys.argv[1] if len(sys.argv) > 1 else "stego6.png"
    end, b64, message = extract(path)
    print("IEND ends at offset:", hex(end))
    print("trailing bytes      :", len(b64))
    print("Base64              :", b64)
    print("decoded message     :", message)


if __name__ == "__main__":
    main()