commit 2fa8f2435f5da18bd103d11797d8cf0132d76803 Author: 404errordeveloper Date: Tue Oct 6 00:25:35 2026 +0200 Initial commit: Null DRM Official Capture, decrypt, and restream toolkit with compiled-in app modules (RTE, TG4, BBC), on-device MITM proxy, streamd control plane, and www. BBC module.yaml is published (clear streams); other module values stay local. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0180f15 --- /dev/null +++ b/.gitignore @@ -0,0 +1,236 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +data/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python + + +# Local dumps / secrets (keep misc/ tracked only as a parking lot) +.DS_Store +traffic.jsonl +captured/ +misc/captured/ +misc/*.txt +misc/*.jsonl +# rteproxy runtime +# rolling live segments +streams/ + +# Capture artifacts (contain auth tokens) +rte_cap.json +.last_capture.json +start_rteproxy.sh + + +# Per-run capture + keys (auth tokens) +outputs/ + + +# Local tool installs from setup-env.sh +bin/ +.cache/ +scripts/env.sh + + +# --- project --- +outputs/ +.cache/ +bin/* +!bin/.gitkeep +data/ +*.log +rte_cap.json +.last_capture.json +# android MITM binaries (rebuild with proxyctl build) +apps/proxy/proxy-android-arm64 +apps/proxy/rteproxy-android-arm64 +# local / VPS deploy + operator scripts (test stuff; keep out of git) +deploy/ +scripts/ +# phone app modules: the Go packages are tracked and compiled into bin/drm.exe, +# but most module.yaml files (package ids, Brightcove keys, origins, KIDs, +# channel maps) are local only and must never be committed. +# Exception: bbc/module.yaml is public (clear streams; package + channel map only). +apps/modules/** +!apps/modules/README.md +!apps/modules/go.mod +!apps/modules/go.sum +!apps/modules/*/ +!apps/modules/*/*.go +!apps/modules/*/**/*.go +!apps/modules/bbc/module.yaml +# local forgejo / deploy secrets +**/.env +**/.push-token diff --git a/README.md b/README.md new file mode 100644 index 0000000..a2f3c9c --- /dev/null +++ b/README.md @@ -0,0 +1,154 @@ +# DRM-Decryption + +Capture Widevine live sessions from Android streaming apps and restream them. + +Everything is one Go binary — `drm` — with every app module compiled in. Each +streaming app (RTE, TG4, …) is a Go package under `apps/modules/`; its secrets and +IDs live in a local, gitignored values file that you fill in yourself. + +- **[docs/](docs/)** — architecture, module authoring, per-provider setup +- **[docs/quickstart.md](docs/quickstart.md)** — the longer version of what follows + +--- + +## Quick start + +### 1. Prerequisites + +| Need | Why | +|---|---| +| **Go 1.25+** | builds everything | +| **Python 3.10+** | `apps/wvkey/wvkey.py` talks to the Widevine CDM | +| **adb** (platform-tools) on `PATH` | only for phone capture | +| A `.wvd` Widevine device file | only for fetching keys | + +Phone capture additionally needs a rooted Android device (Magisk) so the MITM CA +can be injected into the system trust store. Catalog lookups need no phone. + +### 2. Clone and build + +```bash +git clone https://git.nulldrm.com/nulldrm/Null-DRM-Official.git +cd Null-DRM-Official + +# Linux / macOS +go -C apps/cli build -o ../../bin/drm . + +# Windows +go -C apps/cli build -o ../../bin/drm.exe . +``` + +Check it: + +```bash +./bin/drm help +./bin/drm modules # which app modules this build contains +``` + +`drm modules` works immediately, but channel lists stay empty until you add +values — that is the next step. + +### 3. Python CDM helper + +```bash +python -m venv .venv +.venv/bin/pip install -r apps/wvkey/requirements.txt # Windows: .venv\Scripts\pip +``` + +Put your Widevine device file in `data/` (gitignored), e.g. `data/device.wvd`. + +### 4. Fill in a module's values + +Compiled-in module code carries **no** account IDs, policy keys, video IDs, +license URLs or key IDs. Those go in a gitignored file per module, which you create: + +```text +apps/modules/tg4/module.yaml +apps/modules/rte/module.yaml +``` + +Where each value comes from: + +- **[docs/providers/tg4.md](docs/providers/tg4.md)** — Brightcove account ID, policy key, video IDs +- **[docs/providers/rte.md](docs/providers/rte.md)** — license URL, origin hosts, channel KIDs +- **[docs/capture.md](docs/capture.md)** — how to discover all of it for a *new* app + +Any value can be passed per-run instead of stored: + +```bash +./bin/drm catalog --app tg4 --channel ioi --tg4.policy-key 'BCpkAD...' +TG4_POLICY_KEY='BCpkAD...' ./bin/drm catalog --app tg4 --channel ioi +``` + +### 5. Run something + +```bash +# keys from a public catalog — no phone needed +./bin/drm catalog --app tg4 --channel ioi --keys --wvd data/device.wvd + +# phone capture: launch the app, drive it to a channel, grab license + keys +./bin/drm proxy build # once: cross-compile the on-device MITM +./bin/drm capture --app rte --channel rteone --auto-play --wvd data/device.wvd + +# control plane + dashboard on http://127.0.0.1:8083 +./bin/drm serve --bind 127.0.0.1:8083 --data .cache/streamd --token SECRET + +# always-on refresher: watches streamd, re-captures dead channels +STREAMD_TOKEN=SECRET ./bin/drm agent run --config apps/agent/agent.yaml +``` + +Results land in `outputs///`, with `latest/` kept as a copy. + +--- + +## Subcommands + +| Command | What it does | +|---|---| +| `drm modules` | list compiled-in app modules, their channels and config source | +| `drm capture` | one-shot phone MITM capture → `outputs///` | +| `drm catalog` | resolve a channel from its public catalog, no phone | +| `drm serve` | streamd: REST API + dashboard + media supervisor | +| `drm agent` | `run` / `status` / `devices` / `enqueue` / `cancel` | +| `drm proxy` | `build` / `push` / `install-ca` / `start` / `stop` / `discover` / `pull` | + +Run any of them with `--help`. + +## Layout + +```text +apps/cli/ the binary -> bin/drm +apps/modules/ app modules: Go (tracked) + module.yaml (gitignored) +apps/pkg/ shared, provider-neutral libraries +apps/capture/ phone capture command +apps/agent/ always-on key refresher +apps/streamd/ control plane: API, SQLite, dashboard, supervisor +apps/proxy/ MITM host CLI + on-device proxy source +apps/wvkey/ wvkey.py — CDM only +docs/ architecture, authoring, provider setup +data/ secrets: .wvd, CA (gitignored) +outputs/ capture sessions (gitignored) +bin/ built binaries (gitignored) +www/ static site +``` + +## Tests + +```bash +go -C apps/pkg test ./... +go -C apps/modules test ./... +go -C apps/streamd test ./... +``` + +Tests that hit a live origin are behind a build tag, so they stay out of the +normal run: + +```bash +go -C apps/modules test -tags live ./rte/ -v +``` + +## Scope + +For use only on services you are authorised to access, with content you have the +right to decrypt. [docs/architecture.md](docs/architecture.md) describes what the +system does and where it stops. diff --git a/apps/agent/README.md b/apps/agent/README.md new file mode 100644 index 0000000..f68f9d1 --- /dev/null +++ b/apps/agent/README.md @@ -0,0 +1,63 @@ +# agent + +Always-on credentials refresher, hosted by the single binary as `drm agent`. + +```text +apps/agent/ + agentcmd/ the command body, imported by apps/cli + internal/ poller, durable queue, device pool, worker, ws client + agent.yaml config +``` + +## What it does + +1. Polls streamd (`GET /api/streams`) +2. When an **enabled** stream is down/errored or missing `mpd`/`key`, enqueues a job + in a durable SQLite queue under `data_dir` +3. Claims a free ADB device that **already has** the app installed — it never installs +4. Runs a phone capture through the stream's app module +5. `POST /api/streams/:id/credentials` +6. Force-stops the app and clears the device proxy so the phone is free + +It never runs a downloader. Claims are kept alive by a websocket heartbeat, so they +expire by themselves if the agent dies. + +## Run + +```bash +# terminal 1 — control plane +./bin/drm serve --bind 127.0.0.1:8083 --data .cache/streamd --token SECRET + +# terminal 2 — agent +export STREAMD_TOKEN=SECRET +./bin/drm agent run --config apps/agent/agent.yaml +``` + +```bash +./bin/drm agent status +./bin/drm agent devices +./bin/drm agent enqueue --stream tg4-ioi --reason manual +./bin/drm agent cancel --job 3 +``` + +App modules are compiled into the binary, so there is no `modules_dir` to set. The +agent drives whichever module a stream names and contains no provider logic. + +## Config + +`agent.yaml` — `streamd_url`, `token` (or `STREAMD_TOKEN`), `poll_interval_sec`, +`workers`, `max_attempts`, `capture_wait_sec`, `devices` (empty = any authorised +device), `wvd`, `user_agent`, `agent_id`, `data_dir`. + +`wvd` must be set or key fetches fail. + +## Build + +```bash +go -C apps/cli build -o ../../bin/drm . + +# cross-compile +GOOS=darwin GOARCH=arm64 go -C apps/cli build -o ../../bin/drm . +``` + +**Full guide: [docs/streamd.md](../../docs/streamd.md)** diff --git a/apps/agent/agent.yaml b/apps/agent/agent.yaml new file mode 100644 index 0000000..2dbd41a --- /dev/null +++ b/apps/agent/agent.yaml @@ -0,0 +1,18 @@ +# Always-on credentials agent (apps/agent → bin/agent.exe) +# Default path: apps/agent/agent.yaml +streamd_url: http://127.0.0.1:8083 +# Must match streamd --token / STREAMD_TOKEN (mutating API: claim, credentials). +token: "SECRET" +# Stable id across restarts (default = hostname). Required for reclaim after crash. +# agent_id: my-agent +poll_interval_sec: 30 +data_dir: .cache/agent +workers: 1 # concurrent ADB jobs +max_attempts: 5 +capture_wait_sec: 180 +devices: [] # empty = all adb "device" serials; never installs apps +# wvd: data/device.wvd # path to .wvd (required for key fetch) +# user_agent: "" # optional license-request User-Agent +# App modules (rte, tg4, …) are compiled into the binary — there is no +# modules directory to point at. Their local values live in +# apps/modules//module.yaml; see apps/modules/README.md. diff --git a/apps/agent/agentcmd/agentcmd.go b/apps/agent/agentcmd/agentcmd.go new file mode 100644 index 0000000..4325901 --- /dev/null +++ b/apps/agent/agentcmd/agentcmd.go @@ -0,0 +1,259 @@ +// Package agentcmd is the agent CLI, exposed as a library so the single drm +// binary can host it as a subcommand. +package agentcmd + +import ( + "flag" + "fmt" + "log" + "os" + "os/signal" + "strconv" + "syscall" + "text/tabwriter" + + "drmdecryption/apps/agent/internal/client" + "drmdecryption/apps/agent/internal/config" + "drmdecryption/apps/agent/internal/device" + "drmdecryption/apps/agent/internal/poller" + "drmdecryption/apps/agent/internal/queue" + "drmdecryption/apps/agent/internal/worker" + "drmdecryption/apps/agent/internal/wsclient" +) + +// Run dispatches an agent subcommand. args[0] is the subcommand name. +func Run(args []string) error { + log.SetFlags(log.LstdFlags | log.Lmsgprefix) + log.SetPrefix("agent: ") + + if len(args) < 1 { + Usage() + return fmt.Errorf("agent: subcommand required") + } + sub, rest := args[0], args[1:] + switch sub { + case "run": + runCmd(sub, rest) + case "status": + statusCmd(sub, rest) + case "devices": + devicesCmd(sub, rest) + case "enqueue": + enqueueCmd(rest) + case "cancel": + cancelCmd(rest) + case "help", "-h", "--help": + Usage() + default: + Usage() + return fmt.Errorf("agent: unknown subcommand %q", sub) + } + return nil +} + +// Usage prints the agent subcommand help. +func Usage() { + fmt.Fprintf(os.Stderr, `agent — always-on credentials refresher (queue + ADB) + +Usage: + agent run [--config apps/agent/agent.yaml] + agent status [--config ...] + agent devices [--config ...] + agent enqueue --stream NAME [--reason manual] [--config ...] + agent cancel --job ID [--config ...] + +Polls streamd for enabled streams that are down / missing credentials, +enqueues jobs, claims a free phone that already has the app, captures +via MITM, POSTs credentials, then force-stops the app. + +Does not install apps. Does not run NRE/ffmpeg. +`) +} + +func loadCfg(sub string, args []string) (config.Config, *flag.FlagSet) { + fs := flag.NewFlagSet(sub, flag.ExitOnError) + cfgPath := fs.String("config", "", "path to apps/agent/agent.yaml") + _ = fs.Parse(args) + cfg, err := config.Load(*cfgPath) + if err != nil { + log.Fatal(err) + } + return cfg, fs +} + +func openQueue(cfg config.Config) *queue.Store { + q, err := queue.Open(cfg.DataDir) + if err != nil { + log.Fatalf("queue: %v", err) + } + return q +} + +func runCmd(sub string, args []string) { + cfg, _ := loadCfg(sub, args) + api := client.New(cfg.StreamdURL, cfg.Token) + if err := api.Health(); err != nil { + log.Fatalf("streamd unreachable at %s: %v", cfg.StreamdURL, err) + } + q := openQueue(cfg) + defer q.Close() + + if n, err := q.RecoverStaleRunning(); err != nil { + log.Fatalf("recover jobs: %v", err) + } else if n > 0 { + log.Printf("re-queued %d stuck running job(s) from previous run", n) + } + + pool := device.New("", cfg.Devices) + stop := make(chan struct{}) + + p := &poller.Poller{ + API: api, + Queue: q, + MaxAttempts: cfg.MaxAttempts, + Interval: cfg.PollInterval(), + Log: log.Default(), + } + go p.Loop(stop) + + ws := wsclient.New(cfg.StreamdURL, cfg.Token, cfg.AgentID, log.Default()) + go ws.Loop() + + for i := 0; i < cfg.Workers; i++ { + w := &worker.Worker{ + ID: i + 1, + API: api, + Queue: q, + Pool: pool, + AgentID: cfg.AgentID, + CaptureWait: cfg.CaptureWait(), + Python: cfg.Python, + WVD: cfg.WVD, + UserAgent: cfg.UserAgent, + Log: log.Default(), + } + go w.Loop(stop) + } + + log.Printf("running agent_id=%s streamd=%s data=%s workers=%d poll=%s", + cfg.AgentID, cfg.StreamdURL, cfg.DataDir, cfg.Workers, cfg.PollInterval()) + log.Printf("ws heartbeat enabled — claims auto-expire if this agent dies") + log.Printf("waiting for down / missing-credential streams… Ctrl+C to stop") + + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt, syscall.SIGTERM) + <-ch + log.Printf("shutting down…") + ws.Stop() + close(stop) +} + +func statusCmd(sub string, args []string) { + cfg, _ := loadCfg(sub, args) + q := openQueue(cfg) + defer q.Close() + jobs, err := q.ListRecent(40) + if err != nil { + log.Fatal(err) + } + tw := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "ID\tSTREAM\tAPP\tCHANNEL\tSTATUS\tATTEMPTS\tDEVICE\tREASON\tERROR") + for _, j := range jobs { + fmt.Fprintf(tw, "%d\t%s\t%s\t%s\t%s\t%d/%d\t%s\t%s\t%s\n", + j.ID, j.StreamName, j.App, j.Channel, j.Status, j.Attempts, j.MaxAttempts, + j.DeviceSerial, j.Reason, trunc(j.LastError, 60)) + } + _ = tw.Flush() +} + +func devicesCmd(sub string, args []string) { + cfg, _ := loadCfg(sub, args) + pool := device.New("", cfg.Devices) + list, err := pool.List() + if err != nil { + log.Fatal(err) + } + if len(list) == 0 { + fmt.Println("no adb devices") + return + } + tw := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "SERIAL\tSTATE\tMODEL\tALLOWED\tBUSY\tBUSY_FOR") + for _, d := range list { + fmt.Fprintf(tw, "%s\t%s\t%s\t%v\t%v\t%s\n", + d.Serial, d.State, d.Model, d.Allowed, d.Busy, d.BusyFor) + } + _ = tw.Flush() +} + +func enqueueCmd(args []string) { + fs := flag.NewFlagSet("enqueue", flag.ExitOnError) + cfgPath := fs.String("config", "", "path to apps/agent/agent.yaml") + name := fs.String("stream", "", "streamd stream name") + reason := fs.String("reason", "manual", "job reason") + _ = fs.Parse(args) + if *name == "" { + log.Fatal("--stream required") + } + cfg, err := config.Load(*cfgPath) + if err != nil { + log.Fatal(err) + } + api := client.New(cfg.StreamdURL, cfg.Token) + streams, err := api.ListStreams() + if err != nil { + log.Fatal(err) + } + var st *client.Stream + for i := range streams { + if streams[i].Name == *name { + st = &streams[i] + break + } + } + if st == nil { + log.Fatalf("stream %q not found on streamd", *name) + } + q := openQueue(cfg) + defer q.Close() + j, added, err := q.EnqueueIfIdle(st.ID, st.Name, st.App, st.Channel, *reason, cfg.MaxAttempts) + if err != nil { + log.Fatal(err) + } + if !added { + fmt.Printf("already has queued/running job for %s\n", st.Name) + return + } + fmt.Printf("enqueued job=%d stream=%s app=%s channel=%s\n", j.ID, j.StreamName, j.App, j.Channel) +} + +func cancelCmd(args []string) { + fs := flag.NewFlagSet("cancel", flag.ExitOnError) + cfgPath := fs.String("config", "", "path to apps/agent/agent.yaml") + jobID := fs.String("job", "", "job id") + _ = fs.Parse(args) + if *jobID == "" { + log.Fatal("--job required") + } + id, err := strconv.ParseInt(*jobID, 10, 64) + if err != nil { + log.Fatal(err) + } + cfg, err := config.Load(*cfgPath) + if err != nil { + log.Fatal(err) + } + q := openQueue(cfg) + defer q.Close() + if err := q.Cancel(id); err != nil { + log.Fatal(err) + } + fmt.Println("cancelled", id) +} + +func trunc(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} diff --git a/apps/agent/go.mod b/apps/agent/go.mod new file mode 100644 index 0000000..c3aa50f --- /dev/null +++ b/apps/agent/go.mod @@ -0,0 +1,25 @@ +module drmdecryption/apps/agent + +go 1.25.0 + +require ( + drmdecryption v0.0.0 + github.com/gorilla/websocket v1.5.3 + gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.34.5 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + go.starlark.net v0.0.0-20260930220527-d7438c5a85ac // indirect + golang.org/x/sys v0.42.0 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect +) + +replace drmdecryption => ../pkg diff --git a/apps/agent/go.sum b/apps/agent/go.sum new file mode 100644 index 0000000..5ef9a1c --- /dev/null +++ b/apps/agent/go.sum @@ -0,0 +1,55 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +go.starlark.net v0.0.0-20260930220527-d7438c5a85ac h1:pNA9PRXGPFURORAsI3IqfQJ4RLsRXqghCFdvypd/4GY= +go.starlark.net v0.0.0-20260930220527-d7438c5a85ac/go.mod h1:Iue6g6iirlfLoVi/DYCi5/x0h/bAOuWF3dULTKpt2Vo= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/apps/agent/internal/client/client.go b/apps/agent/internal/client/client.go new file mode 100644 index 0000000..b3050db --- /dev/null +++ b/apps/agent/internal/client/client.go @@ -0,0 +1,118 @@ +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +type Client struct { + BaseURL string + Token string + HTTP *http.Client + UserAgent string +} + +func New(baseURL, token string) *Client { + return &Client{ + BaseURL: strings.TrimRight(baseURL, "/"), + Token: token, + HTTP: &http.Client{Timeout: 30 * time.Second}, + UserAgent: "drm-agent/1.0", + } +} + +type Stream struct { + ID int64 `json:"id"` + Name string `json:"name"` + Title string `json:"title"` + App string `json:"app"` + Channel string `json:"channel"` + MPD string `json:"mpd"` + Key string `json:"key"` + Enabled bool `json:"enabled"` + Health string `json:"health"` + LastError string `json:"last_error"` + PlaylistAgeS float64 `json:"playlist_age_s"` + ClaimedBy string `json:"claimed_by"` +} + +type Credentials struct { + MPD string `json:"mpd"` + Key string `json:"key"` + PSSH string `json:"pssh,omitempty"` + Auth string `json:"auth,omitempty"` + PID string `json:"pid,omitempty"` +} + +func (c *Client) Health() error { + _, err := c.do(http.MethodGet, "/api/health", nil) + return err +} + +func (c *Client) ListStreams() ([]Stream, error) { + body, err := c.do(http.MethodGet, "/api/streams", nil) + if err != nil { + return nil, err + } + var wrap struct { + Streams []Stream `json:"streams"` + } + if err := json.Unmarshal(body, &wrap); err != nil { + return nil, err + } + return wrap.Streams, nil +} + +func (c *Client) Claim(id int64, agentID string, ttlSec int) error { + payload := map[string]any{"agent_id": agentID, "ttl_sec": ttlSec} + _, err := c.do(http.MethodPost, fmt.Sprintf("/api/streams/%d/claim", id), payload) + return err +} + +func (c *Client) ReleaseClaim(id int64, agentID string) error { + payload := map[string]any{"agent_id": agentID} + _, err := c.do(http.MethodPost, fmt.Sprintf("/api/streams/%d/claim/release", id), payload) + return err +} + +func (c *Client) PostCredentials(id int64, cred Credentials) error { + _, err := c.do(http.MethodPost, fmt.Sprintf("/api/streams/%d/credentials", id), cred) + return err +} + +func (c *Client) do(method, path string, payload any) ([]byte, error) { + var rdr io.Reader + if payload != nil { + b, err := json.Marshal(payload) + if err != nil { + return nil, err + } + rdr = bytes.NewReader(b) + } + req, err := http.NewRequest(method, c.BaseURL+path, rdr) + if err != nil { + return nil, err + } + if payload != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("User-Agent", c.UserAgent) + if c.Token != "" { + req.Header.Set("Authorization", "Bearer "+c.Token) + } + resp, err := c.HTTP.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("%s %s: %s — %s", method, path, resp.Status, strings.TrimSpace(string(body))) + } + return body, nil +} diff --git a/apps/agent/internal/config/config.go b/apps/agent/internal/config/config.go new file mode 100644 index 0000000..4e42319 --- /dev/null +++ b/apps/agent/internal/config/config.go @@ -0,0 +1,104 @@ +package config + +import ( + "os" + "path/filepath" + "time" + + "gopkg.in/yaml.v3" + + "drmdecryption/repo" +) + +type Config struct { + StreamdURL string `yaml:"streamd_url"` + Token string `yaml:"token"` + PollIntervalSec int `yaml:"poll_interval_sec"` + DataDir string `yaml:"data_dir"` + Workers int `yaml:"workers"` + MaxAttempts int `yaml:"max_attempts"` + CaptureWaitSec int `yaml:"capture_wait_sec"` + Devices []string `yaml:"devices"` + Python string `yaml:"python"` + WVD string `yaml:"wvd"` // path to .wvd device file (required for key fetch) + UserAgent string `yaml:"user_agent"` + AgentID string `yaml:"agent_id"` +} + +func (c Config) PollInterval() time.Duration { + if c.PollIntervalSec <= 0 { + return 30 * time.Second + } + return time.Duration(c.PollIntervalSec) * time.Second +} + +func (c Config) CaptureWait() time.Duration { + if c.CaptureWaitSec <= 0 { + return 180 * time.Second + } + return time.Duration(c.CaptureWaitSec) * time.Second +} + +func Load(path string) (Config, error) { + root := repo.Root() + if path == "" { + path = filepath.Join(root, "apps", "agent", "agent.yaml") + } + var cfg Config + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + cfg = defaults(root) + return cfg, nil + } + return cfg, err + } + if err := yaml.Unmarshal(raw, &cfg); err != nil { + return cfg, err + } + cfg = applyDefaults(cfg, root) + return cfg, nil +} + +func defaults(root string) Config { + return applyDefaults(Config{}, root) +} + +func applyDefaults(cfg Config, root string) Config { + if cfg.StreamdURL == "" { + cfg.StreamdURL = "http://127.0.0.1:8083" + } + if cfg.Token == "" { + cfg.Token = os.Getenv("STREAMD_TOKEN") + } + if cfg.PollIntervalSec <= 0 { + cfg.PollIntervalSec = 30 + } + if cfg.DataDir == "" { + cfg.DataDir = filepath.Join(root, ".cache", "agent") + } else if !filepath.IsAbs(cfg.DataDir) { + cfg.DataDir = filepath.Join(root, cfg.DataDir) + } + if cfg.Workers <= 0 { + cfg.Workers = 1 + } + if cfg.MaxAttempts <= 0 { + cfg.MaxAttempts = 5 + } + if cfg.CaptureWaitSec <= 0 { + cfg.CaptureWaitSec = 180 + } + if cfg.AgentID == "" { + // Stable across restarts so the same agent can reclaim after a crash. + // Override in apps/agent/agent.yaml when running multiple agents. + host, _ := os.Hostname() + if host == "" { + host = "agent" + } + cfg.AgentID = host + } + if cfg.WVD != "" && !filepath.IsAbs(cfg.WVD) { + cfg.WVD = filepath.Join(root, cfg.WVD) + } + return cfg +} diff --git a/apps/agent/internal/device/pool.go b/apps/agent/internal/device/pool.go new file mode 100644 index 0000000..1cc2c9f --- /dev/null +++ b/apps/agent/internal/device/pool.go @@ -0,0 +1,118 @@ +package device + +import ( + "fmt" + "sync" + + "drmdecryption/adb" +) + +// Pool tracks free/busy ADB devices. Never installs apps. +type Pool struct { + mu sync.Mutex + adbBin string + allow map[string]bool // empty allow = all serials + busy map[string]string // serial → job label +} + +func New(adbBin string, allowlist []string) *Pool { + p := &Pool{ + adbBin: adbBin, + allow: map[string]bool{}, + busy: map[string]string{}, + } + for _, s := range allowlist { + if s != "" { + p.allow[s] = true + } + } + return p +} + +type Info struct { + Serial string `json:"serial"` + State string `json:"state"` + Model string `json:"model"` + Busy bool `json:"busy"` + BusyFor string `json:"busy_for,omitempty"` + Allowed bool `json:"allowed"` +} + +func (p *Pool) List() ([]Info, error) { + devs, err := adb.ListDevices(p.adbBin) + if err != nil { + return nil, err + } + p.mu.Lock() + defer p.mu.Unlock() + out := make([]Info, 0, len(devs)) + for _, d := range devs { + allowed := len(p.allow) == 0 || p.allow[d.Serial] + busyFor, busy := p.busy[d.Serial] + out = append(out, Info{ + Serial: d.Serial, + State: d.State, + Model: d.Model, + Busy: busy, + BusyFor: busyFor, + Allowed: allowed, + }) + } + return out, nil +} + +// Acquire finds a free device in state "device" that already has pkg installed. +// Returns ErrWait if none available (do not burn job attempts). +func (p *Pool) Acquire(pkg, jobLabel string) (*adb.Client, string, error) { + devs, err := adb.ListDevices(p.adbBin) + if err != nil { + return nil, "", err + } + base := adb.New() + if p.adbBin != "" { + base.Bin = p.adbBin + } + + p.mu.Lock() + defer p.mu.Unlock() + + var lastMiss string + for _, d := range devs { + if d.State != "device" { + continue + } + if len(p.allow) > 0 && !p.allow[d.Serial] { + continue + } + if _, busy := p.busy[d.Serial]; busy { + continue + } + c := base.WithSerial(d.Serial) + if pkg != "" && !c.PackageInstalled(pkg) { + lastMiss = fmt.Sprintf("%s missing package %s", d.Serial, pkg) + continue + } + p.busy[d.Serial] = jobLabel + return c, d.Serial, nil + } + if lastMiss != "" { + return nil, "", &WaitError{Msg: lastMiss} + } + return nil, "", &WaitError{Msg: "no free adb device"} +} + +func (p *Pool) Release(serial string) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.busy, serial) +} + +// WaitError means the job should stay queued without consuming an attempt. +type WaitError struct{ Msg string } + +func (e *WaitError) Error() string { return e.Msg } + +func IsWait(err error) bool { + _, ok := err.(*WaitError) + return ok +} diff --git a/apps/agent/internal/poller/poller.go b/apps/agent/internal/poller/poller.go new file mode 100644 index 0000000..870b8df --- /dev/null +++ b/apps/agent/internal/poller/poller.go @@ -0,0 +1,111 @@ +package poller + +import ( + "fmt" + "log" + "strings" + "time" + + "drmdecryption/apps/agent/internal/client" + "drmdecryption/apps/agent/internal/queue" +) + +type Poller struct { + API *client.Client + Queue *queue.Store + MaxAttempts int + Interval time.Duration + Log *log.Logger +} + +func (p *Poller) Once() error { + streams, err := p.API.ListStreams() + if err != nil { + return err + } + for _, st := range streams { + if !st.Enabled { + continue + } + reason := needsRefresh(st) + if reason == "" { + continue + } + j, added, err := p.Queue.EnqueueIfIdle(st.ID, st.Name, st.App, st.Channel, reason, p.MaxAttempts) + if err != nil { + return err + } + if added { + p.logf("enqueued job=%d stream=%s reason=%s", j.ID, st.Name, reason) + continue + } + if active, ok, err := p.Queue.ActiveJobForStream(st.ID); err == nil && ok { + extra := "" + if active.Status == queue.StatusQueued && !active.NotBefore.IsZero() && active.NotBefore.After(time.Now().UTC()) { + extra = fmt.Sprintf(" backoff_until=%s", active.NotBefore.Format(time.RFC3339)) + } + p.logf("stream %s needs %s but job=%d already %s (attempts %d/%d)%s", + st.Name, reason, active.ID, active.Status, active.Attempts, active.MaxAttempts, extra) + } + } + return nil +} + +func (p *Poller) Loop(stop <-chan struct{}) { + t := time.NewTicker(p.Interval) + defer t.Stop() + _ = p.Once() + for { + select { + case <-stop: + return + case <-t.C: + if err := p.Once(); err != nil { + p.logf("poll error: %v", err) + } + } + } +} + +func needsRefresh(st client.Stream) string { + if strings.TrimSpace(st.MPD) == "" || strings.TrimSpace(st.Key) == "" { + return "missing_credentials" + } + if !validKeyPair(st.Key) { + return "invalid_credentials" + } + h := strings.ToLower(strings.TrimSpace(st.Health)) + switch h { + case "down", "error", "failed": + return "down" + } + return "" +} + +// validKeyPair accepts KID:KEY as 32-hex:32-hex (colons/dashes in KID ignored). +func validKeyPair(key string) bool { + parts := strings.SplitN(strings.TrimSpace(key), ":", 2) + if len(parts) != 2 { + return false + } + hexOnly := func(s string) string { + var b strings.Builder + for _, r := range s { + switch { + case r >= '0' && r <= '9', r >= 'a' && r <= 'f', r >= 'A' && r <= 'F': + b.WriteRune(r) + } + } + return b.String() + } + kid, k := hexOnly(parts[0]), hexOnly(parts[1]) + return len(kid) == 32 && len(k) == 32 +} + +func (p *Poller) logf(format string, args ...any) { + if p.Log != nil { + p.Log.Printf(format, args...) + return + } + log.Printf(format, args...) +} diff --git a/apps/agent/internal/queue/queue.go b/apps/agent/internal/queue/queue.go new file mode 100644 index 0000000..fbe65c4 --- /dev/null +++ b/apps/agent/internal/queue/queue.go @@ -0,0 +1,334 @@ +package queue + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "time" + + _ "modernc.org/sqlite" +) + +type Status string + +const ( + StatusQueued Status = "queued" + StatusRunning Status = "running" + StatusOK Status = "ok" + StatusFailed Status = "failed" + StatusCancel Status = "cancelled" +) + +type Job struct { + ID int64 + StreamID int64 + StreamName string + App string + Channel string + Reason string + Status Status + Priority int + Attempts int + MaxAttempts int + DeviceSerial string + LastError string + NotBefore time.Time + CreatedAt time.Time + StartedAt time.Time + FinishedAt time.Time +} + +type Store struct { + DB *sql.DB +} + +func Open(dataDir string) (*Store, error) { + if err := os.MkdirAll(dataDir, 0o755); err != nil { + return nil, err + } + path := filepath.Join(dataDir, "agent.db") + dsn := fmt.Sprintf("file:%s?_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)", filepath.ToSlash(path)) + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + s := &Store{DB: db} + if err := s.migrate(); err != nil { + _ = db.Close() + return nil, err + } + return s, nil +} + +func (s *Store) Close() error { return s.DB.Close() } + +func (s *Store) migrate() error { + _, err := s.DB.Exec(` +CREATE TABLE IF NOT EXISTS jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + stream_id INTEGER NOT NULL, + stream_name TEXT NOT NULL DEFAULT '', + app TEXT NOT NULL DEFAULT '', + channel TEXT NOT NULL DEFAULT '', + reason TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'queued', + priority INTEGER NOT NULL DEFAULT 100, + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 5, + device_serial TEXT NOT NULL DEFAULT '', + last_error TEXT NOT NULL DEFAULT '', + not_before TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + started_at TEXT NOT NULL DEFAULT '', + finished_at TEXT NOT NULL DEFAULT '' +); +CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status, priority, id); +CREATE INDEX IF NOT EXISTS idx_jobs_stream ON jobs(stream_id, status); +`) + return err +} + +func now() string { return time.Now().UTC().Format(time.RFC3339) } + +func parseTime(s string) time.Time { + if s == "" { + return time.Time{} + } + t, _ := time.Parse(time.RFC3339, s) + return t +} + +// EnqueueIfIdle inserts a job unless one is already queued/running for the stream. +func (s *Store) EnqueueIfIdle(streamID int64, name, app, channel, reason string, maxAttempts int) (Job, bool, error) { + var existing int + err := s.DB.QueryRow(` +SELECT COUNT(1) FROM jobs WHERE stream_id=? AND status IN ('queued','running')`, streamID).Scan(&existing) + if err != nil { + return Job{}, false, err + } + if existing > 0 { + return Job{}, false, nil + } + if maxAttempts <= 0 { + maxAttempts = 5 + } + res, err := s.DB.Exec(` +INSERT INTO jobs(stream_id,stream_name,app,channel,reason,status,priority,attempts,max_attempts,created_at,not_before) +VALUES(?,?,?,?,?,'queued',100,0,?,?,?)`, + streamID, name, app, channel, reason, maxAttempts, now(), now()) + if err != nil { + return Job{}, false, err + } + id, _ := res.LastInsertId() + j, err := s.Get(id) + return j, true, err +} + +func (s *Store) Get(id int64) (Job, error) { + row := s.DB.QueryRow(` +SELECT id,stream_id,stream_name,app,channel,reason,status,priority,attempts,max_attempts, + device_serial,last_error,not_before,created_at,started_at,finished_at +FROM jobs WHERE id=?`, id) + return scanJob(row) +} + +type scannable interface { + Scan(dest ...any) error +} + +func scanJob(row scannable) (Job, error) { + var j Job + var status, nb, ca, sa, fa string + err := row.Scan( + &j.ID, &j.StreamID, &j.StreamName, &j.App, &j.Channel, &j.Reason, &status, + &j.Priority, &j.Attempts, &j.MaxAttempts, &j.DeviceSerial, &j.LastError, + &nb, &ca, &sa, &fa, + ) + if err != nil { + return j, err + } + j.Status = Status(status) + j.NotBefore = parseTime(nb) + j.CreatedAt = parseTime(ca) + j.StartedAt = parseTime(sa) + j.FinishedAt = parseTime(fa) + return j, nil +} + +// ClaimNext marks the next ready queued job as running. Returns false if none. +func (s *Store) ClaimNext() (Job, bool, error) { + tx, err := s.DB.Begin() + if err != nil { + return Job{}, false, err + } + defer func() { _ = tx.Rollback() }() + + nowStr := now() + row := tx.QueryRow(` +SELECT id FROM jobs +WHERE status='queued' AND (not_before='' OR not_before<=?) +ORDER BY priority ASC, id ASC +LIMIT 1`, nowStr) + var id int64 + if err := row.Scan(&id); err != nil { + if err == sql.ErrNoRows { + return Job{}, false, nil + } + return Job{}, false, err + } + _, err = tx.Exec(`UPDATE jobs SET status='running', started_at=?, last_error='' WHERE id=? AND status='queued'`, nowStr, id) + if err != nil { + return Job{}, false, err + } + if err := tx.Commit(); err != nil { + return Job{}, false, err + } + j, err := s.Get(id) + return j, true, err +} + +func (s *Store) SetDevice(id int64, serial string) error { + _, err := s.DB.Exec(`UPDATE jobs SET device_serial=? WHERE id=?`, serial, id) + return err +} + +func (s *Store) MarkOK(id int64) error { + _, err := s.DB.Exec(`UPDATE jobs SET status='ok', finished_at=?, last_error='' WHERE id=?`, now(), id) + return err +} + +// MarkFailed increments attempts. Re-queues with backoff unless maxed out. +// waitingDevice=true keeps status queued without burning an attempt. +func (s *Store) MarkFailed(id int64, msg string, waitingDevice bool) error { + j, err := s.Get(id) + if err != nil { + return err + } + if waitingDevice { + _, err = s.DB.Exec(` +UPDATE jobs SET status='queued', device_serial='', last_error=?, started_at='' WHERE id=?`, + msg, id) + return err + } + attempts := j.Attempts + 1 + if attempts >= j.MaxAttempts { + _, err = s.DB.Exec(` +UPDATE jobs SET status='failed', attempts=?, last_error=?, finished_at=? WHERE id=?`, + attempts, msg, now(), id) + return err + } + backoff := time.Duration(1< 15*time.Minute { + backoff = 15 * time.Minute + } + nb := time.Now().UTC().Add(backoff).Format(time.RFC3339) + _, err = s.DB.Exec(` +UPDATE jobs SET status='queued', attempts=?, last_error=?, not_before=?, device_serial='', started_at='' WHERE id=?`, + attempts, msg, nb, id) + return err +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func (s *Store) Cancel(id int64) error { + _, err := s.DB.Exec(` +UPDATE jobs SET status='cancelled', finished_at=? WHERE id=? AND status IN ('queued','running')`, now(), id) + return err +} + +func (s *Store) ListRecent(limit int) ([]Job, error) { + if limit <= 0 { + limit = 30 + } + rows, err := s.DB.Query(` +SELECT id,stream_id,stream_name,app,channel,reason,status,priority,attempts,max_attempts, + device_serial,last_error,not_before,created_at,started_at,finished_at +FROM jobs ORDER BY id DESC LIMIT ?`, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Job + for rows.Next() { + j, err := scanJob(rows) + if err != nil { + return nil, err + } + out = append(out, j) + } + return out, rows.Err() +} + +func (s *Store) ActiveForStream(streamID int64) (bool, error) { + var n int + err := s.DB.QueryRow(`SELECT COUNT(1) FROM jobs WHERE stream_id=? AND status IN ('queued','running')`, streamID).Scan(&n) + return n > 0, err +} + +// RecoverStaleRunning re-queues jobs left in status=running after a crash/kill +// so a down stream is not blocked forever by EnqueueIfIdle. +func (s *Store) RecoverStaleRunning() (int, error) { + nowStr := now() + res, err := s.DB.Exec(` +UPDATE jobs SET status='queued', device_serial='', started_at='', + last_error='recovered after agent restart', not_before=? +WHERE status='running'`, nowStr) + if err != nil { + return 0, err + } + n, _ := res.RowsAffected() + return int(n), nil +} + +// NextDelayed returns the soonest queued job that is still waiting on not_before. +func (s *Store) NextDelayed() (Job, time.Duration, bool, error) { + nowStr := now() + row := s.DB.QueryRow(` +SELECT id,stream_id,stream_name,app,channel,reason,status,priority,attempts,max_attempts, + device_serial,last_error,not_before,created_at,started_at,finished_at +FROM jobs +WHERE status='queued' AND not_before!='' AND not_before>? +ORDER BY not_before ASC LIMIT 1`, nowStr) + j, err := scanJob(row) + if err == sql.ErrNoRows { + return Job{}, 0, false, nil + } + if err != nil { + return Job{}, 0, false, err + } + until := time.Until(j.NotBefore) + if until < 0 { + until = 0 + } + return j, until, true, nil +} + +// ClearBackoff makes a queued job runnable immediately (e.g. after fixing auth). +func (s *Store) ClearBackoff(id int64) error { + _, err := s.DB.Exec(`UPDATE jobs SET not_before=? WHERE id=? AND status='queued'`, now(), id) + return err +} + +// ActiveJobForStream returns the queued/running job for a stream, if any. +func (s *Store) ActiveJobForStream(streamID int64) (Job, bool, error) { + row := s.DB.QueryRow(` +SELECT id,stream_id,stream_name,app,channel,reason,status,priority,attempts,max_attempts, + device_serial,last_error,not_before,created_at,started_at,finished_at +FROM jobs WHERE stream_id=? AND status IN ('queued','running') +ORDER BY id DESC LIMIT 1`, streamID) + j, err := scanJob(row) + if err == sql.ErrNoRows { + return Job{}, false, nil + } + if err != nil { + return Job{}, false, err + } + return j, true, nil +} diff --git a/apps/agent/internal/worker/worker.go b/apps/agent/internal/worker/worker.go new file mode 100644 index 0000000..c139b9d --- /dev/null +++ b/apps/agent/internal/worker/worker.go @@ -0,0 +1,149 @@ +package worker + +import ( + "fmt" + "log" + "strings" + "time" + + "drmdecryption/app" + "drmdecryption/phonecap" + + "drmdecryption/apps/agent/internal/client" + "drmdecryption/apps/agent/internal/device" + "drmdecryption/apps/agent/internal/queue" +) + +type Worker struct { + ID int + API *client.Client + Queue *queue.Store + Pool *device.Pool + AgentID string + CaptureWait time.Duration + Python string + WVD string + UserAgent string + Log *log.Logger + + lastDelayLog time.Time +} + +func (w *Worker) Loop(stop <-chan struct{}) { + t := time.NewTicker(2 * time.Second) + defer t.Stop() + for { + select { + case <-stop: + return + case <-t.C: + w.tick() + } + } +} + +func (w *Worker) tick() { + job, ok, err := w.Queue.ClaimNext() + if err != nil { + w.logf("claim next: %v", err) + return + } + if !ok { + if j, until, found, err := w.Queue.NextDelayed(); err == nil && found { + if time.Since(w.lastDelayLog) > 30*time.Second { + w.lastDelayLog = time.Now() + w.logf("no runnable jobs; next is job=%d stream=%s in %s (backoff)", + j.ID, j.StreamName, until.Round(time.Second)) + } + } + return + } + w.logf("job=%d running stream=%s app=%s channel=%s reason=%s", + job.ID, job.StreamName, job.App, job.Channel, job.Reason) + if err := w.runJob(job); err != nil { + if device.IsWait(err) || isAuthErr(err) || isUnknownAppErr(err) { + // Config/environment issues — do not burn attempts or multi-minute backoff. + w.logf("job=%d waiting: %v", job.ID, err) + _ = w.Queue.MarkFailed(job.ID, err.Error(), true) + return + } + w.logf("job=%d failed: %v", job.ID, err) + _ = w.Queue.MarkFailed(job.ID, err.Error(), false) + return + } + _ = w.Queue.MarkOK(job.ID) + w.logf("job=%d ok stream=%s", job.ID, job.StreamName) +} + +func isAuthErr(err error) bool { + if err == nil { + return false + } + s := strings.ToLower(err.Error()) + return strings.Contains(s, "401") || strings.Contains(s, "unauthorized") +} + +func isUnknownAppErr(err error) bool { + if err == nil { + return false + } + s := strings.ToLower(err.Error()) + return strings.Contains(s, "unknown app") +} + +func (w *Worker) runJob(job queue.Job) error { + a, err := app.Get(job.App) + if err != nil { + return err + } + label := fmt.Sprintf("job-%d:%s", job.ID, job.StreamName) + c, serial, err := w.Pool.Acquire(a.Package(), label) + if err != nil { + return err + } + defer w.Pool.Release(serial) + _ = w.Queue.SetDevice(job.ID, serial) + w.logf("job=%d device=%s package=%s", job.ID, serial, a.Package()) + + // Short TTL; /ws/agent heartbeats renew while this process is alive. + if err := w.API.Claim(job.StreamID, w.AgentID, 90); err != nil { + return fmt.Errorf("streamd claim: %w", err) + } + defer func() { _ = w.API.ReleaseClaim(job.StreamID, w.AgentID) }() + + res, err := phonecap.Run(phonecap.Options{ + App: a, + Channel: job.Channel, + Client: c, + Python: w.Python, + WVD: w.WVD, + UserAgent: w.UserAgent, + Wait: w.CaptureWait, + CloseApp: true, + ClearProxy: true, + }) + if err != nil { + return err + } + + cred := client.Credentials{ + MPD: res.MPD, + Key: res.Key, + PSSH: res.Session.PSSH, + Auth: res.Session.Auth, + PID: res.Session.PID, + } + if err := w.API.PostCredentials(job.StreamID, cred); err != nil { + return fmt.Errorf("post credentials: %w", err) + } + return nil +} + +func (w *Worker) logf(format string, args ...any) { + prefix := fmt.Sprintf("worker/%d: ", w.ID) + if w.Log != nil { + w.Log.Printf(prefix+format, args...) + return + } + log.Printf(prefix+format, args...) +} diff --git a/apps/agent/internal/wsclient/wsclient.go b/apps/agent/internal/wsclient/wsclient.go new file mode 100644 index 0000000..aec0cd9 --- /dev/null +++ b/apps/agent/internal/wsclient/wsclient.go @@ -0,0 +1,148 @@ +package wsclient + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" + "strings" + "time" + + "github.com/gorilla/websocket" +) + +// Client keeps a streamd /ws/agent connection alive and sends heartbeats +// so claims auto-renew; on disconnect streamd releases this agent's claims. +type Client struct { + BaseURL string + Token string + AgentID string + Log *log.Logger + + stop chan struct{} +} + +func New(baseURL, token, agentID string, lg *log.Logger) *Client { + if lg == nil { + lg = log.Default() + } + return &Client{ + BaseURL: strings.TrimRight(baseURL, "/"), + Token: token, + AgentID: agentID, + Log: lg, + stop: make(chan struct{}), + } +} + +func (c *Client) Stop() { close(c.stop) } + +// Loop reconnects forever until Stop. +func (c *Client) Loop() { + backoff := time.Second + for { + select { + case <-c.stop: + return + default: + } + if err := c.session(); err != nil { + c.Log.Printf("ws: session ended: %v — retry in %s", err, backoff) + } + select { + case <-c.stop: + return + case <-time.After(backoff): + } + if backoff < 30*time.Second { + backoff *= 2 + } + } +} + +func (c *Client) session() error { + u, err := url.Parse(c.BaseURL) + if err != nil { + return err + } + switch u.Scheme { + case "https": + u.Scheme = "wss" + default: + u.Scheme = "ws" + } + u.Path = "/ws/agent" + q := u.Query() + q.Set("agent_id", c.AgentID) + if c.Token != "" { + q.Set("token", c.Token) + } + u.RawQuery = q.Encode() + + hdr := http.Header{} + if c.Token != "" { + hdr.Set("Authorization", "Bearer "+c.Token) + } + dialer := websocket.Dialer{HandshakeTimeout: 10 * time.Second} + conn, _, err := dialer.Dial(u.String(), hdr) + if err != nil { + return fmt.Errorf("dial: %w", err) + } + defer conn.Close() + c.Log.Printf("ws: connected to %s as %s", u.Host, c.AgentID) + + _ = conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + conn.SetPongHandler(func(string) error { + _ = conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + return nil + }) + + done := make(chan struct{}) + go func() { + defer close(done) + for { + _, data, err := conn.ReadMessage() + if err != nil { + return + } + _ = conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + var msg map[string]any + if json.Unmarshal(data, &msg) == nil { + if t, _ := msg["type"].(string); t == "heartbeat_ack" { + if ok, _ := msg["ok"].(bool); !ok { + c.Log.Printf("ws: heartbeat nack: %v", msg["error"]) + } + } + } + } + }() + + ticker := time.NewTicker(20 * time.Second) + defer ticker.Stop() + // Immediate heartbeat so existing claims renew right away on reconnect. + if err := c.writeHeartbeat(conn); err != nil { + return err + } + for { + select { + case <-c.stop: + _ = conn.WriteJSON(map[string]string{"type": "release_all"}) + return nil + case <-done: + return fmt.Errorf("read closed") + case <-ticker.C: + if err := c.writeHeartbeat(conn); err != nil { + return err + } + } + } +} + +func (c *Client) writeHeartbeat(conn *websocket.Conn) error { + _ = conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) + return conn.WriteJSON(map[string]string{ + "type": "heartbeat", + "agent_id": c.AgentID, + }) +} diff --git a/apps/capture/README.md b/apps/capture/README.md new file mode 100644 index 0000000..4274d50 --- /dev/null +++ b/apps/capture/README.md @@ -0,0 +1,24 @@ +# capture + +One-shot phone MITM capture, hosted by the single binary as `drm capture`. + +```text +apps/capture/ + capturecmd/ the command body, imported by apps/cli +``` + +```bash +./bin/drm capture # passive: play anything +./bin/drm capture --app rte # launch, navigate by hand +./bin/drm capture --app rte --channel rteone --auto-play \ + --wvd data/device.wvd # fully automated +``` + +Writes `outputs///` plus `latest/`. App modules are compiled in, so +there is no modules directory to point at; `drm modules` lists what is available. + +**Full guide: [docs/capture.md](../../docs/capture.md)** — device setup, flags, +discovering a new app, and troubleshooting. + +The catalog lookup that used to be a separate `tg4info` binary is now the +provider-agnostic `drm catalog` (`apps/cli/catalogcmd`). diff --git a/apps/capture/capturecmd/capturecmd.go b/apps/capture/capturecmd/capturecmd.go new file mode 100644 index 0000000..e957e2b --- /dev/null +++ b/apps/capture/capturecmd/capturecmd.go @@ -0,0 +1,99 @@ +// Package capturecmd is the one-shot phone MITM capture command, exposed as a +// library so the single drm binary can host it as a subcommand. +package capturecmd + +import ( + "flag" + "fmt" + "strings" + "time" + + "drmdecryption/adb" + "drmdecryption/app" + "drmdecryption/modcfg" + "drmdecryption/phonecap" +) + +// Run performs one capture. App modules are compiled in, so there is no modules +// directory to scan: --app names a registered provider. +func Run(args []string) error { + fs := flag.NewFlagSet("capture", flag.ExitOnError) + appName := fs.String("app", "", "app module. Empty = passive MITM, no launch/auto-play") + channel := fs.String("channel", "", "channel id for --auto-play") + autoPlay := fs.Bool("auto-play", false, "auto-navigate to --channel (requires --app and --channel)") + waitSec := fs.Int("wait", 180, "seconds to wait for license/mpd capture") + python := fs.String("python", "", "python for wvkey.py (default: .venv)") + wvd := fs.String("wvd", "", "path to .wvd device file (required for key fetch)") + userAgent := fs.String("user-agent", "", "optional User-Agent for license requests") + serial := fs.String("serial", "", "adb device serial (default: sole/USB selected device)") + closeApp := fs.Bool("close", true, "force-stop --app when capture finishes (ignored in passive mode)") + keyMode := fs.String("key-mode", "", "override the app's key mode: modulardrm | raw | none") + // Each module contributes --. overrides for its local values. + app.BindFlags(fs) + fs.Usage = func() { + fmt.Fprintf(fs.Output(), "capture — one-shot phone MITM capture\n\nUsage:\n capture --app <%s> [--channel ID] [--auto-play] [--wvd PATH]\n capture (passive: play anything on the phone)\n\nFlags:\n", strings.Join(app.Names(), "|")) + fs.PrintDefaults() + } + if err := fs.Parse(args); err != nil { + return err + } + + c := adb.New() + if *serial != "" { + c = c.WithSerial(*serial) + } + + // No --app: passive MITM — the operator plays whatever they want and we dump + // whatever DRM traffic appears. + if strings.TrimSpace(*appName) == "" { + _, err := phonecap.RunPassive(phonecap.Options{ + Client: c, + Python: *python, + WVD: *wvd, + UserAgent: *userAgent, + Wait: time.Duration(*waitSec) * time.Second, + KeyMode: *keyMode, + ClearProxy: true, + }) + return err + } + + a, err := app.Get(*appName) + if err != nil { + return err + } + if *autoPlay && strings.TrimSpace(*channel) == "" { + return fmt.Errorf("--auto-play requires --channel (%s)", knownChannels(a)) + } + if *channel != "" && !a.HasChannel(*channel) { + return fmt.Errorf("unknown channel %q for %s (%s)", *channel, a.Name(), knownChannels(a)) + } + + _, err = phonecap.Run(phonecap.Options{ + App: a, + Channel: *channel, + Client: c, + Python: *python, + WVD: *wvd, + UserAgent: *userAgent, + Wait: time.Duration(*waitSec) * time.Second, + NoAutoPlay: !*autoPlay, + CloseApp: *closeApp, + KeyMode: *keyMode, + ClearProxy: true, + }) + return err +} + +// knownChannels describes what the operator can pick, or where to declare it when +// the module has no local values file yet. +func knownChannels(a app.App) string { + ids := []string{} + for _, ch := range a.Channels() { + ids = append(ids, ch.ID) + } + if len(ids) == 0 { + return "no channels configured — add " + modcfg.Path(a.Name()) + } + return "known: " + strings.Join(ids, ", ") +} diff --git a/apps/capture/go.mod b/apps/capture/go.mod new file mode 100644 index 0000000..af6edc7 --- /dev/null +++ b/apps/capture/go.mod @@ -0,0 +1,13 @@ +module drmdecryption/apps/capture + +go 1.25.0 + +require drmdecryption v0.0.0 + +require ( + go.starlark.net v0.0.0-20260930220527-d7438c5a85ac // indirect + golang.org/x/sys v0.42.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace drmdecryption => ../pkg diff --git a/apps/capture/go.sum b/apps/capture/go.sum new file mode 100644 index 0000000..908fb03 --- /dev/null +++ b/apps/capture/go.sum @@ -0,0 +1,12 @@ +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +go.starlark.net v0.0.0-20260930220527-d7438c5a85ac h1:pNA9PRXGPFURORAsI3IqfQJ4RLsRXqghCFdvypd/4GY= +go.starlark.net v0.0.0-20260930220527-d7438c5a85ac/go.mod h1:Iue6g6iirlfLoVi/DYCi5/x0h/bAOuWF3dULTKpt2Vo= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/apps/cli/catalogcmd/catalogcmd.go b/apps/cli/catalogcmd/catalogcmd.go new file mode 100644 index 0000000..377c3c6 --- /dev/null +++ b/apps/cli/catalogcmd/catalogcmd.go @@ -0,0 +1,181 @@ +// Package catalogcmd resolves a channel's stream credentials from its provider's +// public catalog, with no phone involved. It is provider-agnostic: any app module +// implementing app.Catalog works here. +package catalogcmd + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "sort" + "strings" + + "drmdecryption/app" + "drmdecryption/repo" + "drmdecryption/session" +) + +// Run performs a catalog lookup for one channel. +func Run(args []string) error { + fs := flag.NewFlagSet("catalog", flag.ExitOnError) + appName := fs.String("app", "", "app module to query") + channel := fs.String("channel", "", "channel id") + asJSON := fs.Bool("json", false, "print streamd-ready JSON only") + list := fs.Bool("list", false, "list the app's channels and their catalog ids") + writeOut := fs.Bool("out", false, "write outputs///session.json") + withKeys := fs.Bool("keys", false, "also fetch PSSH and resolve KID:KEY via the local CDM") + python := fs.String("python", "", "python for wvkey.py (default: .venv)") + wvd := fs.String("wvd", "", "path to .wvd device file (required with --keys)") + // Each module contributes --. overrides for its local values. + app.BindFlags(fs) + fs.Usage = func() { + fmt.Fprintf(fs.Output(), "catalog — resolve a channel from its public catalog (no phone)\n\nUsage:\n catalog --app <%s> --channel ID [--keys] [--json]\n\nFlags:\n", strings.Join(catalogApps(), "|")) + fs.PrintDefaults() + } + if err := fs.Parse(args); err != nil { + return err + } + + available := catalogApps() + if strings.TrimSpace(*appName) == "" { + if len(available) != 1 { + return fmt.Errorf("--app required (apps with a catalog: %s)", strings.Join(available, ", ")) + } + *appName = available[0] + } + a, err := app.Get(*appName) + if err != nil { + return err + } + cat, ok := a.(app.Catalog) + if !ok { + return fmt.Errorf("app %q has no catalog — capture it from the phone instead: drm capture --app %s", a.Name(), a.Name()) + } + + if *list { + return listChannels(a, cat) + } + if strings.TrimSpace(*channel) == "" { + return fmt.Errorf("--channel required (see: drm catalog --app %s --list)", a.Name()) + } + + info, err := cat.Resolve(*channel) + if err != nil { + return err + } + if *withKeys { + fmt.Fprintln(os.Stderr, "[*] fetching PSSH + Widevine keys…") + if err := cat.EnrichWithKeys(&info, *python, *wvd); err != nil { + return err + } + fmt.Fprintf(os.Stderr, "[+] key %s (%d total)\n", info.Key, len(info.Keys)) + } + if *writeOut || *withKeys { + dir, err := session.WriteStream(repo.Root(), a.Name(), info) + if err != nil { + return err + } + fmt.Fprintf(os.Stderr, "[+] wrote %s\n", dir) + } + if *asJSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(info) + } + printInfo(a.Name(), info) + return nil +} + +// catalogApps lists compiled-in modules that can answer catalog lookups. +func catalogApps() []string { + out := []string{} + for _, a := range app.All() { + if _, ok := a.(app.Catalog); ok { + out = append(out, a.Name()) + } + } + sort.Strings(out) + return out +} + +func listChannels(a app.App, cat app.Catalog) error { + if l, ok := cat.(app.CatalogLister); ok { + rows, flags, err := l.ListChannels() + if err != nil { + return err + } + fmt.Printf("%-10s %-16s %s\n", "CHANNEL", "CATALOG ID", "LABEL") + sort.Slice(rows, func(i, j int) bool { return rows[i].ID < rows[j].ID }) + for _, r := range rows { + id := r.CatalogID + if id == "" { + id = "-" + } + fmt.Printf("%-10s %-16s %s\n", r.ID, id, r.Label) + } + if len(flags) > 0 { + keys := make([]string, 0, len(flags)) + for k := range flags { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, fmt.Sprintf("%s=%v", k, flags[k])) + } + fmt.Printf("\nprovider flags: %s\n", strings.Join(parts, " ")) + } + return nil + } + // Fall back to the plain channel list every app can produce. + fmt.Printf("%-10s %s\n", "CHANNEL", "LABEL") + for _, ch := range a.Channels() { + fmt.Printf("%-10s %s\n", ch.ID, ch.Label) + } + return nil +} + +func printInfo(appName string, info session.Stream) { + fmt.Printf("name: %s\n", info.Name) + fmt.Printf("title: %s\n", info.Title) + fmt.Printf("app/channel: %s / %s\n", info.App, info.Channel) + if info.VideoID != "" { + fmt.Printf("catalog id: %s\n", info.VideoID) + } + fmt.Printf("manifest: %s\n", info.MPD) + fmt.Printf("manifest type: %s\n", info.ManifestType) + fmt.Printf("license_url: %s\n", truncate(info.LicenseURL, 100)) + if info.PSSH != "" { + fmt.Printf("pssh: %s\n", truncate(info.PSSH, 64)) + } + if info.Key != "" { + fmt.Printf("key: %s\n", info.Key) + } + for i, k := range info.Keys { + if k == info.Key { + continue + } + fmt.Printf("key[%d]: %s\n", i, k) + } + body, _ := json.MarshalIndent(map[string]any{ + "name": info.Name, + "title": info.Title, + "app": info.App, + "channel": info.Channel, + "mpd": info.MPD, + "key": info.Key, + "headers_json": info.HeadersJSON, + "rewriter": info.Rewriter, + }, "", " ") + fmt.Printf("\nstreamd create body:\n%s\n", body) + fmt.Printf("\nTip: drm catalog --app %s --channel %s --keys --wvd path/to/device.wvd --json\n", + appName, info.Channel) +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "..." +} diff --git a/apps/cli/go.mod b/apps/cli/go.mod new file mode 100644 index 0000000..3b37ed2 --- /dev/null +++ b/apps/cli/go.mod @@ -0,0 +1,36 @@ +module drmdecryption/apps/cli + +go 1.25.0 + +require ( + drmdecryption v0.0.0 + drmdecryption/apps/agent v0.0.0 + drmdecryption/apps/capture v0.0.0 + drmdecryption/apps/proxy v0.0.0 + drmdecryption/apps/streamd v0.0.0 + drmdecryption/modules v0.0.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.3 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.42.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/sqlite v1.34.5 // indirect +) + +replace ( + drmdecryption => ../pkg + drmdecryption/apps/agent => ../agent + drmdecryption/apps/capture => ../capture + drmdecryption/apps/proxy => ../proxy + drmdecryption/apps/streamd => ../streamd + drmdecryption/modules => ../modules +) diff --git a/apps/cli/go.sum b/apps/cli/go.sum new file mode 100644 index 0000000..58f3755 --- /dev/null +++ b/apps/cli/go.sum @@ -0,0 +1,49 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/apps/cli/main.go b/apps/cli/main.go new file mode 100644 index 0000000..c72d009 --- /dev/null +++ b/apps/cli/main.go @@ -0,0 +1,76 @@ +// Command drm is the single entry point for the toolchain: phone capture, +// catalog lookups, the always-on agent, the streamd control plane and the +// on-device MITM. Every app module under apps/modules is compiled in. +package main + +import ( + "fmt" + "os" + + "drmdecryption/apps/agent/agentcmd" + "drmdecryption/apps/capture/capturecmd" + "drmdecryption/apps/cli/catalogcmd" + "drmdecryption/apps/cli/modulescmd" + "drmdecryption/apps/proxy/proxyctlcmd" + "drmdecryption/apps/streamd/servecmd" + + // Links every app module into this binary. + _ "drmdecryption/modules/all" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + cmd, args := os.Args[1], os.Args[2:] + var err error + switch cmd { + case "capture": + err = capturecmd.Run(args) + case "catalog": + err = catalogcmd.Run(args) + case "modules": + err = modulescmd.Run(args) + case "agent": + err = agentcmd.Run(args) + case "serve": + err = servecmd.Run(append([]string{"serve"}, args...)) + case "proxy": + err = proxyctlcmd.Run(args) + case "help", "-h", "--help": + usage() + return + default: + usage() + err = fmt.Errorf("unknown command %q", cmd) + } + if err != nil { + fmt.Fprintln(os.Stderr, "[!]", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `drm — DRM capture / restream toolchain + +Usage: + drm modules list compiled-in app modules and their channels + drm capture --app NAME [--channel ID] [--auto-play] [--wvd PATH] + one-shot phone MITM capture + drm catalog --app NAME --channel ID [--keys] [--json] [--list] + resolve a channel from its public catalog (no phone) + drm agent run|status|devices|enqueue|cancel [--config …] + always-on credential refresher + drm serve [--bind …] [--data …] [--token …] + streamd control plane + dashboard + drm proxy build|push|install-ca|start|stop|discover|pull|clear-proxy + on-device HTTPS MITM + +App modules are compiled in; their local values live in +apps/modules//module.yaml (gitignored) and can be overridden per run with +--. flags or _ environment variables. + +Run any subcommand with --help for its own flags. +`) +} diff --git a/apps/cli/modulescmd/modulescmd.go b/apps/cli/modulescmd/modulescmd.go new file mode 100644 index 0000000..5e0db5e --- /dev/null +++ b/apps/cli/modulescmd/modulescmd.go @@ -0,0 +1,78 @@ +// Package modulescmd lists the app modules compiled into this binary, their +// channels, and where each one's local values came from. +package modulescmd + +import ( + "flag" + "fmt" + "os" + "sort" + "text/tabwriter" + + "drmdecryption/app" + "drmdecryption/modcfg" +) + +// Run prints the compiled-in app modules. +func Run(args []string) error { + fs := flag.NewFlagSet("modules", flag.ExitOnError) + verbose := fs.Bool("channels", true, "list each module's channels") + if err := fs.Parse(args); err != nil { + return err + } + + names := app.Names() + if len(names) == 0 { + return fmt.Errorf("no app modules compiled in — check the blank imports in apps/modules/all/all.go") + } + tw := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "APP\tPACKAGE\tKEY MODE\tREWRITER\tCHANNELS\tVALUES") + for _, name := range names { + a, err := app.Get(name) + if err != nil { + fmt.Fprintf(tw, "%s\t-\t-\t-\t-\t%v\n", name, err) + continue + } + rewriter := "none" + if sd, ok := a.(app.StreamDefaults); ok && sd.RewriterName() != "" { + rewriter = sd.RewriterName() + } + pkg := a.Package() + if pkg == "" { + pkg = "(not configured)" + } + values := modcfg.Path(name) + if _, err := os.Stat(values); err != nil { + values = "missing: " + values + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%d\t%s\n", + name, pkg, a.KeyMode(), rewriter, len(a.Channels()), values) + } + _ = tw.Flush() + + if !*verbose { + return nil + } + for _, name := range names { + a, err := app.Get(name) + if err != nil { + continue + } + chans := a.Channels() + sort.Slice(chans, func(i, j int) bool { return chans[i].ID < chans[j].ID }) + fmt.Printf("\n%s channels:", name) + if len(chans) == 0 { + fmt.Printf(" none — add %s\n", modcfg.Path(name)) + continue + } + fmt.Println() + for _, ch := range chans { + catalog := "" + if _, ok := a.(app.Catalog); ok { + catalog = " (catalog)" + } + fmt.Printf(" %-10s %s%s\n", ch.ID, ch.Label, catalog) + } + } + return nil +} diff --git a/apps/modules/README.md b/apps/modules/README.md new file mode 100644 index 0000000..e518fd5 --- /dev/null +++ b/apps/modules/README.md @@ -0,0 +1,67 @@ +# App modules + +One Go package per streaming app. Every module is **compiled into the binary** — +there is no plugin loading and no modules directory to point a binary at. + +```text +apps/modules/ Go module: drmdecryption/modules + go.mod replace drmdecryption => ../pkg + all/all.go blank-imports every module — the link point + provider/ shared base: values loading, channel aliases, durations + /*.go TRACKED: the app's logic + /module.yaml usually GITIGNORED values (bbc is published) +``` + +## Tracked code, untracked values + +This split is the rule for modules with secrets; BBC is the exception (clear +streams — package id + channel map only, so `bbc/module.yaml` is committed): + +| `/*.go` (tracked) | `/module.yaml` (gitignored except bbc) | +|---|---| +| launch + auto-play sequence | android package id | +| navigation idioms | license URL, origin hostnames | +| manifest rewrite shape | channel KIDs, account id, policy key | +| catalog request flow | video ids, playback config URL | +| which capture fields are required | UI selectors, labels, aliases | + +Module Go source contains **no** account id, policy key, video id, license URL, +origin host or KID. Modules with secrets have a test asserting an empty config +yields empty credentials: + +```bash +go -C apps/modules test ./... -run NoHardcoded -v +``` + +Values arrive by **flag → environment → module.yaml → Go default**, and only +mechanical defaults (timeouts, DASH timescales, card geometry) live in code: + +```bash +./bin/drm catalog --app tg4 --channel ioi --tg4.policy-key BCpkAD... +export TG4_POLICY_KEY=BCpkAD... # same thing +``` + +## Adding a module + +1. Create `apps/modules/myapp/` with `config.go` and `myapp.go` +2. Register from `init()`: `appreg.Register(Name, New)` and + `appreg.RegisterFlags(bindFlags)` +3. Add one line to `all/all.go`: `_ "drmdecryption/modules/myapp"` +4. Create `apps/modules/myapp/module.yaml` with your values +5. `go -C apps/cli build -o ../../bin/drm .` then `./bin/drm modules` + +**Authoring guide: [docs/modules.md](../../docs/modules.md)** — the full contract, +`uiflow` reference, optional capability interfaces, and a checklist. + +**Finding the values:** [docs/capture.md](../../docs/capture.md) for a new app, +[docs/providers/](../../docs/providers/) for the modules already here. + +## Run + +```bash +./bin/drm modules # what is compiled in +./bin/drm capture --app rte --channel rteone --auto-play +./bin/drm capture --app bbc --channel bbcone # transparent MITM; play on phone; MPD only +./bin/drm catalog --app tg4 --channel ioi --keys --wvd data/device.wvd +./bin/drm agent run --config apps/agent/agent.yaml +``` diff --git a/apps/modules/all/all.go b/apps/modules/all/all.go new file mode 100644 index 0000000..33b94a3 --- /dev/null +++ b/apps/modules/all/all.go @@ -0,0 +1,10 @@ +// Package all links every app module into the binary. Importing it for side +// effects is what makes `--app ` work; adding a provider means adding one +// line here. +package all + +import ( + _ "drmdecryption/modules/bbc" + _ "drmdecryption/modules/rte" + _ "drmdecryption/modules/tg4" +) diff --git a/apps/modules/all/all_test.go b/apps/modules/all/all_test.go new file mode 100644 index 0000000..fd6113b --- /dev/null +++ b/apps/modules/all/all_test.go @@ -0,0 +1,72 @@ +package all + +import ( + "flag" + "testing" + + appreg "drmdecryption/app" +) + +// Every module must be constructible with no module.yaml present — a fresh clone +// has none, and the binary still has to start and list its providers. +func TestAllModulesRegisterAndConstruct(t *testing.T) { + names := appreg.Names() + if len(names) == 0 { + t.Fatal("no app modules registered — check the blank imports in all.go") + } + for _, n := range names { + a, err := appreg.Get(n) + if err != nil { + t.Fatalf("construct %s: %v", n, err) + } + if a.Name() != n { + t.Errorf("%s: Name() = %q", n, a.Name()) + } + switch a.KeyMode() { + case "raw", "modulardrm", "none": + default: + t.Errorf("%s: KeyMode() = %q, want raw, modulardrm, or none", n, a.KeyMode()) + } + if a.CAHash() == "" { + t.Errorf("%s: CAHash() empty", n) + } + if a.ProxyBin() == "" { + t.Errorf("%s: ProxyBin() empty", n) + } + if a.MPDRewriter() == nil { + t.Errorf("%s: MPDRewriter() nil", n) + } + if a.CaptureHints().RemoteLog == "" { + t.Errorf("%s: CaptureHints has no remote log", n) + } + } +} + +// Flag binding must not construct any app, so overrides land before config load. +func TestBindFlagsRegistersOverrides(t *testing.T) { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + appreg.BindFlags(fs) + count := 0 + fs.VisitAll(func(*flag.Flag) { count++ }) + if count == 0 { + t.Fatal("no module override flags registered") + } +} + +// A module declaring a rewriter must have registered it under that name. +func TestDeclaredRewritersAreRegistered(t *testing.T) { + for _, a := range appreg.All() { + sd, ok := a.(appreg.StreamDefaults) + if !ok { + continue + } + name := sd.RewriterName() + if name == "" || name == "none" { + continue + } + if a.MPDRewriter().Name() != name { + t.Errorf("%s: RewriterName()=%q but MPDRewriter().Name()=%q", + a.Name(), name, a.MPDRewriter().Name()) + } + } +} diff --git a/apps/modules/bbc/bbc.go b/apps/modules/bbc/bbc.go new file mode 100644 index 0000000..22d5bcf --- /dev/null +++ b/apps/modules/bbc/bbc.go @@ -0,0 +1,129 @@ +// Package bbc is the BBC iPlayer app module: clear DASH/HLS captured through +// transparent MITM (UK VPN stays on). Only the manifest URL is required — no +// Widevine keys for the mobile-phone-main mediaset streams we capture. +package bbc + +import ( + "flag" + "fmt" + "os" + "time" + + "drmdecryption/adb" + appreg "drmdecryption/app" + "drmdecryption/capture" + "drmdecryption/modules/provider" + "drmdecryption/mpd" + "drmdecryption/uiflow" +) + +func init() { + appreg.Register(Name, New) + appreg.RegisterFlags(bindFlags) +} + +func bindFlags(fs *flag.FlagSet) { + fs.StringVar(&flags.packageName, Name+".package", "", "override "+Name+" android package id") + fs.StringVar(&flags.caHash, Name+".ca-hash", "", "override "+Name+" MITM CA subject hash") + fs.StringVar(&flags.proxyBin, Name+".proxy-bin", "", "override "+Name+" on-device MITM binary") +} + +// App is the BBC iPlayer plugin. +type App struct { + *provider.Base + cfg Config +} + +// New constructs the plugin from module.yaml + flags + env. +func New() (appreg.App, error) { + cfg, res, err := loadConfig() + if err != nil { + return nil, err + } + labels := make(map[string]string, len(cfg.Channels)) + for id, ch := range cfg.Channels { + labels[id] = ch.Label + } + return &App{Base: provider.New(Name, cfg.Common, labels, res), cfg: cfg}, nil +} + +// KeyMode none: mobile streams are clear (CDN signed URLs); no wvkey. +func (a *App) KeyMode() string { return "none" } + +// CaptureHints: only the MPD/HLS master from mediaselector is required. +func (a *App) CaptureHints() capture.Hints { + return a.Hints("mpd") +} + +// UseTransparentMITM: NordVPN UK conflicts with Wi‑Fi http_proxy. +func (a *App) UseTransparentMITM() bool { return true } + +// MPDRewriter: passthrough — no rewrite needed for clear BBC DASH. +func (a *App) MPDRewriter() mpd.Rewriter { return mpd.Passthrough{} } + +// Channels lists configured live targets. +func (a *App) Channels() []appreg.Channel { + out := make([]appreg.Channel, 0, len(a.cfg.Channels)) + for _, id := range a.ChannelIDs() { + out = append(out, appreg.Channel{ID: id, Label: a.Label(id)}) + } + return out +} + +// Launch cold-starts iPlayer. +func (a *App) Launch(c *adb.Client) error { + c.EnsureAwake() + fmt.Printf("[*] Launching %s (%s)…\n", Name, a.Package()) + c.ForceStop(a.Package()) + time.Sleep(400 * time.Millisecond) + if err := uiflow.MonkeyLaunch(c, a.Package()); err != nil { + return err + } + time.Sleep(time.Duration(a.cfg.Timeouts.LaunchSettle)) + c.DismissShadeIfFocused() + if err := uiflow.WaitUI(c, a.CacheDir(), uiflow.Match{ + DescContains: a.cfg.UI.LaunchDescContains, + ResourceContains: a.cfg.UI.LaunchResourceContains, + }, time.Duration(a.cfg.Timeouts.LaunchWaitUI)); err != nil { + fmt.Fprintf(os.Stderr, "[!] launch UI wait: %v — continue and play manually\n", err) + } + return nil +} + +// AutoPlay: best-effort; iPlayer UI varies — operator can play by hand. +func (a *App) AutoPlay(c *adb.Client, channel string) error { + id, err := a.Resolve(channel) + if err != nil { + return err + } + ch := a.cfg.Channels[id] + fmt.Printf("[*] Auto-play %s — open Live and start playback on the phone…\n", a.Label(id)) + if ch.VPID != "" { + fmt.Printf(" expected mediaselector vpid ≈ %s\n", ch.VPID) + } + c.DismissShadeIfFocused() + if len(ch.PlayDesc) == 0 && len(ch.ChipDesc) == 0 { + return fmt.Errorf("no UI selectors for %s — play manually on the phone", id) + } + if len(ch.ChipDesc) > 0 { + if err := uiflow.TapUI(c, a.CacheDir(), uiflow.Match{DescRE: ch.ChipDesc}, 20*time.Second); err != nil { + return fmt.Errorf("channel chip: %w", err) + } + time.Sleep(2 * time.Second) + } + if len(ch.PlayDesc) > 0 { + if err := uiflow.TapUI(c, a.CacheDir(), uiflow.Match{DescRE: ch.PlayDesc}, 15*time.Second); err != nil { + return fmt.Errorf("play: %w", err) + } + } + return uiflow.WaitPlayback(c, a.CacheDir(), a.Package(), uiflow.PlaybackOpts{ + Timeout: time.Duration(a.cfg.Timeouts.Playback), + }) +} + +// StreamDefaults for clear live DASH. +func (a *App) VideoSelect() string { return "res=1280x720:for=best" } +func (a *App) AudioSelect() string { return "lang=en:for=best" } +func (a *App) RewriterName() string { return "none" } +func (a *App) LiveWaitSeconds() int { return 2 } +func (a *App) TSReadyBytes() int64 { return 256 << 10 } diff --git a/apps/modules/bbc/bbc_test.go b/apps/modules/bbc/bbc_test.go new file mode 100644 index 0000000..1ea2bca --- /dev/null +++ b/apps/modules/bbc/bbc_test.go @@ -0,0 +1,80 @@ +package bbc + +import ( + "testing" + + appreg "drmdecryption/app" + "drmdecryption/capture" + "drmdecryption/modcfg" + "drmdecryption/modules/provider" + "drmdecryption/mpd" +) + +var ( + _ appreg.App = (*App)(nil) + _ appreg.StreamDefaults = (*App)(nil) + _ appreg.TransparentMITM = (*App)(nil) +) + +func testApp(t *testing.T) *App { + t.Helper() + cfg := Config{ + Channels: map[string]Channel{ + "bbcone": {Label: "BBC One", VPID: "bbc_one_london"}, + "bbctwo": {Label: "BBC Two", VPID: "bbc_two_england"}, + }, + Common: provider.Common{ + Package: "bbc.iplayer.android", + Aliases: map[string]string{"one": "bbcone", "bbc1": "bbcone"}, + Score: capture.ScoreCfg{ + Hosts: []string{"open.live.bbc.co.uk", "vs-cmaf-push-uk"}, + }, + }, + }.withDefaults() + labels := map[string]string{} + for id, ch := range cfg.Channels { + labels[id] = ch.Label + } + return &App{Base: provider.New(Name, cfg.Common, labels, modcfg.Result{Path: "test"}), cfg: cfg} +} + +func TestNoHardcodedSecrets(t *testing.T) { + cfg := Config{}.withDefaults() + if cfg.Package != "" || cfg.LicenseURL != "" { + t.Fatalf("empty config must not invent package/license: %+v", cfg) + } +} + +func TestKeyModeNoneAndMPDOnly(t *testing.T) { + a := testApp(t) + if a.KeyMode() != "none" { + t.Fatalf("KeyMode = %q, want none", a.KeyMode()) + } + h := a.CaptureHints() + if len(h.Require) != 1 || h.Require[0] != "mpd" { + t.Fatalf("Require = %v, want [mpd]", h.Require) + } + if !a.UseTransparentMITM() { + t.Fatal("BBC must use transparent MITM (UK VPN)") + } + if _, ok := a.MPDRewriter().(mpd.Passthrough); !ok { + t.Fatal("expected Passthrough rewriter") + } +} + +func TestAliases(t *testing.T) { + a := testApp(t) + id, err := a.Resolve("one") + if err != nil || id != "bbcone" { + t.Fatalf("Resolve(one) = %q %v", id, err) + } +} + +func TestCaptureHintsScoreHosts(t *testing.T) { + a := testApp(t) + h := a.CaptureHints() + u := "https://vs-cmaf-push-uk.live.fastly.md.bbci.co.uk/x/mobile.mpd" + if h.Score.Score(u) <= 0 { + t.Fatalf("expected positive score for %s (score=%d)", u, h.Score.Score(u)) + } +} diff --git a/apps/modules/bbc/config.go b/apps/modules/bbc/config.go new file mode 100644 index 0000000..1961a22 --- /dev/null +++ b/apps/modules/bbc/config.go @@ -0,0 +1,74 @@ +package bbc + +import ( + "time" + + "drmdecryption/modcfg" + "drmdecryption/modules/provider" +) + +// Name is the module id (--app bbc). +const Name = "bbc" + +// Channel is one live (or catch-up) target. Phone UI selectors are optional — +// capture works with manual play; auto-play uses them when present. +type Channel struct { + Label string `yaml:"label"` + // VPID is the mediaselector version/service id (e.g. bbc_one_london). + VPID string `yaml:"vpid"` + // ChipDesc / PlayDesc are optional UI matchers for --auto-play. + ChipDesc []string `yaml:"chip_desc"` + PlayDesc []string `yaml:"play_desc"` +} + +// Config is apps/modules/bbc/module.yaml. +type Config struct { + provider.Common `yaml:",inline"` + + Channels map[string]Channel `yaml:"channels"` + + Timeouts Timeouts `yaml:"timeouts"` + UI UI `yaml:"ui"` +} + +// Timeouts for launch / optional autoplay. +type Timeouts struct { + LaunchWaitUI provider.Duration `yaml:"launch_wait_ui"` + LaunchSettle provider.Duration `yaml:"launch_settle"` + Playback provider.Duration `yaml:"playback"` +} + +// UI markers for launch readiness (localized; keep loose). +type UI struct { + LaunchDescContains []string `yaml:"launch_desc_contains"` + LaunchResourceContains []string `yaml:"launch_resource_contains"` +} + +func (c Config) withDefaults() Config { + t := &c.Timeouts + t.LaunchWaitUI = provider.Duration(t.LaunchWaitUI.D(30 * time.Second)) + t.LaunchSettle = provider.Duration(t.LaunchSettle.D(4 * time.Second)) + t.Playback = provider.Duration(t.Playback.D(60 * time.Second)) + if len(c.UI.LaunchDescContains) == 0 { + c.UI.LaunchDescContains = []string{"iPlayer", "Home", "Live"} + } + return c +} + +var flags struct { + packageName string + caHash string + proxyBin string +} + +func loadConfig() (Config, modcfg.Result, error) { + var cfg Config + res, err := modcfg.Load(Name, &cfg) + if err != nil { + return cfg, res, err + } + cfg.Package = modcfg.Override(Name, "package", flags.packageName, cfg.Package) + cfg.CAHash = modcfg.Override(Name, "ca_hash", flags.caHash, cfg.CAHash) + cfg.ProxyBin = modcfg.Override(Name, "proxy_bin", flags.proxyBin, cfg.ProxyBin) + return cfg.withDefaults(), res, nil +} diff --git a/apps/modules/bbc/module.yaml b/apps/modules/bbc/module.yaml new file mode 100644 index 0000000..38a749e --- /dev/null +++ b/apps/modules/bbc/module.yaml @@ -0,0 +1,51 @@ +# BBC iPlayer — published values. Package id and channel map only; +# no Widevine license URL (mobile-phone-main streams are clear). + +package: bbc.iplayer.android +proxy_bin: bin/proxy-android-arm64 +ca_hash: "6c3578b4" + +# Manifest-URL scoring for the on-device MITM / waiter. +score: + hosts: + - open.live.bbc.co.uk + - vs-cmaf-push-uk + - vod-dash-uk + - akamaized.net + - bbci.co.uk + - bidi.net.uk + - bidi.bbc.co.uk + deny: + - telemetry.api.bbci + - bag.api.bbc + - ibl.api.bbci + - thumbnail + +aliases: + one: bbcone + bbc1: bbcone + london: bbcone + two: bbctwo + bbc2: bbctwo + news: bbcnews + four: bbcfour + +channels: + bbcone: + label: BBC One + vpid: bbc_one_london + bbctwo: + label: BBC Two + vpid: bbc_two_england + bbcnews: + label: BBC News + vpid: bbc_news_channel + bbcfour: + label: BBC Four + vpid: bbc_four + +# timeouts: +# launch_wait_ui: 30s +# playback: 60s +# ui: +# launch_desc_contains: ["iPlayer", "Home", "Live"] diff --git a/apps/modules/go.mod b/apps/modules/go.mod new file mode 100644 index 0000000..6635e2a --- /dev/null +++ b/apps/modules/go.mod @@ -0,0 +1,9 @@ +module drmdecryption/modules + +go 1.25.0 + +require drmdecryption v0.0.0 + +require gopkg.in/yaml.v3 v3.0.1 // indirect + +replace drmdecryption => ../pkg diff --git a/apps/modules/go.sum b/apps/modules/go.sum new file mode 100644 index 0000000..a62c313 --- /dev/null +++ b/apps/modules/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/apps/modules/provider/duration.go b/apps/modules/provider/duration.go new file mode 100644 index 0000000..d542f57 --- /dev/null +++ b/apps/modules/provider/duration.go @@ -0,0 +1,41 @@ +package provider + +import ( + "fmt" + "time" +) + +// Duration is a time.Duration that accepts the spellings a module.yaml uses: +// a Go duration string ("25s", "1200ms") or a bare number meaning seconds. +// yaml.v3 decodes time.Duration only as raw nanoseconds, which no one writes. +type Duration time.Duration + +// D returns the value as a time.Duration, or def when unset. +func (d Duration) D(def time.Duration) time.Duration { + if d == 0 { + return def + } + return time.Duration(d) +} + +func (d *Duration) UnmarshalYAML(unmarshal func(any) error) error { + var s string + if err := unmarshal(&s); err == nil { + if s == "" { + *d = 0 + return nil + } + v, err := time.ParseDuration(s) + if err != nil { + return fmt.Errorf("invalid duration %q: %w", s, err) + } + *d = Duration(v) + return nil + } + var f float64 + if err := unmarshal(&f); err != nil { + return fmt.Errorf("duration must be a string like \"25s\" or a number of seconds") + } + *d = Duration(time.Duration(f * float64(time.Second))) + return nil +} diff --git a/apps/modules/provider/provider.go b/apps/modules/provider/provider.go new file mode 100644 index 0000000..8936d48 --- /dev/null +++ b/apps/modules/provider/provider.go @@ -0,0 +1,147 @@ +// Package provider holds the plumbing every app module shares: loading its local +// values file, channel id normalization and aliasing, and the boilerplate half of +// app.App. Module packages embed Base and add their own navigation and catalog. +package provider + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "drmdecryption/capture" + "drmdecryption/modcfg" + "drmdecryption/phonecap" + "drmdecryption/repo" +) + +// Common is the part of a module.yaml that every module has. Modules embed it in +// their own config struct with `yaml:",inline"`. +type Common struct { + Package string `yaml:"package"` + LicenseURL string `yaml:"license_url"` + ProxyBin string `yaml:"proxy_bin"` + CAHash string `yaml:"ca_hash"` + Aliases map[string]string `yaml:"aliases"` + Score capture.ScoreCfg `yaml:"score"` +} + +// Base implements the boilerplate half of app.App. +type Base struct { + name string + common Common + labels map[string]string // canonical channel id -> label + cacheDir string + cfg modcfg.Result +} + +// New builds a Base. labels maps canonical channel ids to display labels. +func New(name string, common Common, labels map[string]string, cfg modcfg.Result) *Base { + root := repo.Root() + cacheDir := filepath.Join(root, ".cache", "ui-"+name) + _ = os.MkdirAll(cacheDir, 0o755) + return &Base{name: name, common: common, labels: labels, cacheDir: cacheDir, cfg: cfg} +} + +func (b *Base) Name() string { return b.name } +func (b *Base) Package() string { return b.common.Package } +func (b *Base) LicenseURL() string { return b.common.LicenseURL } +func (b *Base) CacheDir() string { return b.cacheDir } + +// ConfigPath / ConfigLoaded report where this module's values came from. +func (b *Base) ConfigPath() string { return b.cfg.Path } +func (b *Base) ConfigLoaded() bool { return b.cfg.Loaded } + +// ProxyBin resolves the on-device MITM binary, relative paths against repo root. +func (b *Base) ProxyBin() string { + if b.common.ProxyBin != "" { + return modcfg.Resolve(b.common.ProxyBin) + } + root := repo.Root() + for _, p := range []string{ + filepath.Join(root, "bin", "proxy-android-arm64"), + filepath.Join(root, "apps", "proxy", "proxy-android-arm64"), + } { + if st, err := os.Stat(p); err == nil && !st.IsDir() { + return p + } + } + return filepath.Join(root, "bin", "proxy-android-arm64") +} + +func (b *Base) CAHash() string { + if b.common.CAHash == "" { + return phonecap.DefaultCAHash + } + return b.common.CAHash +} + +// Score is this module's manifest-URL scoring, merged onto the neutral baseline. +func (b *Base) Score() capture.ScoreCfg { + return capture.DefaultScoreCfg().Merge(b.common.Score) +} + +// Hints returns device-layout hints with this module's scoring applied. +func (b *Base) Hints(require ...string) capture.Hints { + h := capture.DefaultHints() + h.Require = require + h.Score = b.Score() + return h +} + +// NormalizeID folds a channel id to its comparison form. +func NormalizeID(id string) string { + id = strings.ToLower(strings.TrimSpace(id)) + for _, ch := range []string{"-", "_", " "} { + id = strings.ReplaceAll(id, ch, "") + } + return id +} + +// Resolve maps any spelling or alias of a channel to its canonical id. +func (b *Base) Resolve(id string) (string, error) { + want := NormalizeID(id) + for canon := range b.labels { + if NormalizeID(canon) == want { + return canon, nil + } + } + for alias, canon := range b.common.Aliases { + if NormalizeID(alias) == want { + if _, ok := b.labels[canon]; ok { + return canon, nil + } + } + } + known := make([]string, 0, len(b.labels)) + for k := range b.labels { + known = append(known, k) + } + return "", fmt.Errorf("unknown channel %q for %s (known: %s) — add it to %s", + id, b.name, strings.Join(known, "|"), b.cfg.Path) +} + +func (b *Base) HasChannel(id string) bool { + _, err := b.Resolve(id) + return err == nil +} + +// Label returns a channel's display label, falling back to its id. +func (b *Base) Label(canonID string) string { + if l := b.labels[canonID]; l != "" { + return l + } + return canonID +} + +// ChannelIDs lists canonical channel ids, sorted so listings and API responses +// are stable (map iteration order is not). +func (b *Base) ChannelIDs() []string { + out := make([]string, 0, len(b.labels)) + for k := range b.labels { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/apps/modules/rte/config.go b/apps/modules/rte/config.go new file mode 100644 index 0000000..c67b99d --- /dev/null +++ b/apps/modules/rte/config.go @@ -0,0 +1,118 @@ +package rte + +import ( + "time" + + "drmdecryption/modcfg" + "drmdecryption/modules/provider" + "drmdecryption/mpd" +) + +// Name is the module id (--app rte). +const Name = "rte" + +// Channel is one live channel's phone-UI selectors and encoder identity. All of +// it is data: nothing here is compiled into the binary. +type Channel struct { + Label string `yaml:"label"` + // ChipDesc matches the channel chip on the Live tab. + ChipDesc []string `yaml:"chip_desc"` + // PlayDesc matches the large hero Play button. + PlayDesc []string `yaml:"play_desc"` +} + +// Config is apps/modules/rte/module.yaml. +type Config struct { + provider.Common `yaml:",inline"` + + Channels map[string]Channel `yaml:"channels"` + + // Origins maps an encoder channel id to its Smooth Streaming origin base. + Origins map[string]string `yaml:"origins"` + // KIDs maps an encoder channel id to its Widevine KID. + KIDs map[string]string `yaml:"kids"` + // EncoderChannelRE matches origin channel ids that use the encoder timeline. + EncoderChannelRE string `yaml:"encoder_channel_re"` + // OriginMarker is the path element identifying an origin BaseURL. + OriginMarker string `yaml:"origin_marker"` + + // Synthetic tunes the generated MPD. Everything in it has a safe default. + Synthetic mpd.SyntheticConfig `yaml:"synthetic"` + + // HeroMinArea is the smallest node area accepted as the hero Play button. + HeroMinArea int `yaml:"hero_min_area"` + + Timeouts Timeouts `yaml:"timeouts"` + UI UI `yaml:"ui"` +} + +// Timeouts for the launch / autoplay sequence. Written as "25s" in module.yaml. +type Timeouts struct { + LaunchWaitUI provider.Duration `yaml:"launch_wait_ui"` + LiveTab provider.Duration `yaml:"live_tab"` + ChipWait provider.Duration `yaml:"chip_wait"` + PlayWait provider.Duration `yaml:"play_wait"` + Playback provider.Duration `yaml:"playback"` + LaunchSettle provider.Duration `yaml:"launch_settle"` + AfterLiveTab provider.Duration `yaml:"after_live_tab"` + AfterChip provider.Duration `yaml:"after_chip"` +} + +// UI holds the markers the launch/autoplay sequence waits on, as config so an +// app UI redesign needs no rebuild. +type UI struct { + // LaunchDescContains: any of these means the app is up. + LaunchDescContains []string `yaml:"launch_desc_contains"` + // LiveTabDescRE matches the Live tab to tap. + LiveTabDescRE []string `yaml:"live_tab_desc_re"` +} + +func (c Config) withDefaults() Config { + if c.EncoderChannelRE == "" { + c.EncoderChannelRE = `^(vc|channel)\d+$` + } + if c.OriginMarker == "" { + c.OriginMarker = ".isml" + } + if c.HeroMinArea == 0 { + c.HeroMinArea = 200_000 + } + t := &c.Timeouts + t.LaunchWaitUI = provider.Duration(t.LaunchWaitUI.D(25 * time.Second)) + t.LiveTab = provider.Duration(t.LiveTab.D(30 * time.Second)) + t.ChipWait = provider.Duration(t.ChipWait.D(20 * time.Second)) + t.PlayWait = provider.Duration(t.PlayWait.D(15 * time.Second)) + t.Playback = provider.Duration(t.Playback.D(45 * time.Second)) + t.LaunchSettle = provider.Duration(t.LaunchSettle.D(4 * time.Second)) + t.AfterLiveTab = provider.Duration(t.AfterLiveTab.D(2 * time.Second)) + t.AfterChip = provider.Duration(t.AfterChip.D(2 * time.Second)) + if len(c.UI.LaunchDescContains) == 0 { + c.UI.LaunchDescContains = []string{"live tab", "header logo"} + } + if len(c.UI.LiveTabDescRE) == 0 { + c.UI.LiveTabDescRE = []string{"Live tab"} + } + c.Synthetic = c.Synthetic.WithDefaults() + return c +} + +// flag overrides, bound before any app is constructed. +var flags struct { + packageName string + licenseURL string + caHash string + proxyBin string +} + +func loadConfig() (Config, modcfg.Result, error) { + var cfg Config + res, err := modcfg.Load(Name, &cfg) + if err != nil { + return cfg, res, err + } + cfg.Package = modcfg.Override(Name, "package", flags.packageName, cfg.Package) + cfg.LicenseURL = modcfg.Override(Name, "license_url", flags.licenseURL, cfg.LicenseURL) + cfg.CAHash = modcfg.Override(Name, "ca_hash", flags.caHash, cfg.CAHash) + cfg.ProxyBin = modcfg.Override(Name, "proxy_bin", flags.proxyBin, cfg.ProxyBin) + return cfg.withDefaults(), res, nil +} diff --git a/apps/modules/rte/live_test.go b/apps/modules/rte/live_test.go new file mode 100644 index 0000000..b89e50a --- /dev/null +++ b/apps/modules/rte/live_test.go @@ -0,0 +1,130 @@ +//go:build live + +// Live tests hit the real origin and need this module's local module.yaml. +// They are excluded from the normal test run; enable with: +// +// go -C apps/modules test -tags live ./rte/ -v +package rte + +import ( + "regexp" + "strconv" + "strings" + "testing" + + "drmdecryption/mpd" +) + +// The rewriter must publish a usable manifest from the KID alone — this is the +// path taken when the ad-stitched upstream session has expired (HTTP 410) and all +// that is left is the key from an earlier capture. +func TestLiveRewriteFromKIDOnly(t *testing.T) { + cfg, res, err := loadConfig() + if err != nil { + t.Fatal(err) + } + if !res.Loaded { + t.Skipf("no local values at %s", res.Path) + } + if len(cfg.KIDs) == 0 { + t.Skip("no kids configured") + } + + for channel, kid := range cfg.KIDs { + t.Run(channel, func(t *testing.T) { + resetCaches(channel) + r := NewRewriterWith(cfg) + + // Empty upstream = expired session; only the KID hint is available. + out, err := r.Rewrite(nil, kid) + if err != nil { + t.Fatalf("rewrite from KID: %v", err) + } + got := string(out) + + origin := cfg.Origins[channel] + if origin == "" { + t.Fatalf("no origin configured for %s", channel) + } + for _, want := range []string{ + `type="dynamic"`, + origin + cfg.Synthetic.SegmentPathSuffix, + channel + "-" + cfg.Synthetic.VideoName, + channel + "-" + cfg.Synthetic.AudioName, + } { + if !strings.Contains(got, want) { + t.Errorf("manifest missing %q", want) + } + } + u, _ := mpd.KIDToUUID(kid) + if !strings.Contains(got, `cenc:default_KID="`+u+`"`) { + t.Errorf("manifest does not carry KID %s", u) + } + + // The timeline must be non-empty and on the encoder grid, or the + // downloader asks the origin for segments that do not exist. + re := regexp.MustCompile(``) + ms := re.FindAllStringSubmatch(got, -1) + if len(ms) != 2 { + t.Fatalf("want 2 timelines, got %d", len(ms)) + } + grid, err := mpd.LearnGrid(channel, origin, cfg.Synthetic.GridSpec()) + if err != nil { + t.Fatalf("learn grid: %v", err) + } + for i, m := range ms { + start, _ := strconv.ParseInt(m[1], 10, 64) + dur, _ := strconv.ParseInt(m[2], 10, 64) + rep, _ := strconv.Atoi(m[3]) + wantDur, mod := grid.VDur, grid.VMod + if i == 1 { + wantDur, mod = grid.ADur, grid.AMod + } + if dur != wantDur { + t.Errorf("timeline %d: d=%d, want %d (from the live origin)", i, dur, wantDur) + } + if int64(rep+1) != cfg.Synthetic.WindowSegments { + t.Errorf("timeline %d: %d segments, want %d", i, rep+1, cfg.Synthetic.WindowSegments) + } + if off := ((start-mod)%wantDur + wantDur) % wantDur; off != 0 { + t.Errorf("timeline %d: start %d is off the encoder grid by %d", i, start, off) + } + if start <= 0 { + t.Errorf("timeline %d: non-positive start %d", i, start) + } + } + t.Logf("%s: %d bytes, vdur=%d adur=%d", channel, len(out), grid.VDur, grid.ADur) + }) + } +} + +// The KID→channel→origin lookup is what makes the expired-session path work. +func TestLiveKIDResolvesToConfiguredOrigin(t *testing.T) { + cfg, res, err := loadConfig() + if err != nil { + t.Fatal(err) + } + if !res.Loaded { + t.Skipf("no local values at %s", res.Path) + } + r := NewRewriterWith(cfg) + for channel, kid := range cfg.KIDs { + gotCh, gotBase, ok := r.ResolveKID(mpd.KIDHex(kid)) + if !ok { + t.Errorf("%s: KID %s did not resolve", channel, kid) + continue + } + if gotCh != channel { + t.Errorf("KID %s resolved to %q, want %q", kid, gotCh, channel) + } + if gotBase != cfg.Origins[channel] { + t.Errorf("%s: origin %q, want %q", channel, gotBase, cfg.Origins[channel]) + } + } +} + +func resetCaches(channel string) { + // The grid/anchor caches live in pkg/mpd and are keyed by channel; a fresh + // process per test run is enough, so nothing to do here beyond documenting it. + _ = channel +} diff --git a/apps/modules/rte/mpd.go b/apps/modules/rte/mpd.go new file mode 100644 index 0000000..2d5a31b --- /dev/null +++ b/apps/modules/rte/mpd.go @@ -0,0 +1,146 @@ +package rte + +import ( + "fmt" + "regexp" + "strings" + "sync" + + "drmdecryption/mpd" +) + +func init() { + mpd.Register(Name, func() mpd.Rewriter { return NewRewriter() }) +} + +var reBaseURL = regexp.MustCompile(`(?i)]*>([^<]+)`) + +// Rewriter turns an ad-stitched upstream MPD into a synthetic manifest on the +// origin encoder's own segment grid, so a downloader keeps pulling live segments +// after the upstream session expires. All origins and KIDs come from config. +type Rewriter struct { + cfg Config + + mu sync.Mutex + fallbackCh string + fallbackBase string +} + +// NewRewriter loads the module config and returns a rewriter. A config error is +// deferred to Rewrite so registry lookup never fails at startup. +func NewRewriter() *Rewriter { + cfg, _, err := loadConfig() + if err != nil { + return &Rewriter{cfg: Config{}.withDefaults()} + } + return &Rewriter{cfg: cfg} +} + +// NewRewriterWith builds a rewriter from an already-loaded config. +func NewRewriterWith(cfg Config) *Rewriter { + return &Rewriter{cfg: cfg} +} + +func (r *Rewriter) Name() string { return Name } + +// Prime records a channel/origin learned elsewhere (e.g. from the KID). +func (r *Rewriter) Prime(channel, originBase string) { + r.mu.Lock() + defer r.mu.Unlock() + if channel != "" { + r.fallbackCh = channel + } + if originBase != "" { + r.fallbackBase = originBase + } +} + +// ResolveKID maps a KID back to its encoder channel and origin. +func (r *Rewriter) ResolveKID(kidHex string) (channel, originBase string, ok bool) { + for ch, kid := range r.cfg.KIDs { + if mpd.KIDHex(kid) == kidHex { + return ch, r.cfg.Origins[ch], true + } + } + return "", "", false +} + +// Rewrite produces the synthetic MPD. upstreamXML may be empty (expired session): +// the KID fallback then supplies the channel and origin. +func (r *Rewriter) Rewrite(upstreamXML []byte, kidHint string) ([]byte, error) { + r.mu.Lock() + fbCh, fbBase := r.fallbackCh, r.fallbackBase + r.mu.Unlock() + + payload, channel, originBase, err := r.rewrite(string(upstreamXML), kidHint, fbCh, fbBase) + if channel != "" || originBase != "" { + r.Prime(channel, originBase) + } + return payload, err +} + +func (r *Rewriter) rewrite(xmlText, kidHint, fallbackChannel, fallbackBase string) (payload []byte, channel, originBase string, err error) { + cfg := r.cfg + channel, originBase = r.originFromXML(xmlText) + resolvedKid := mpd.ExtractDefaultKID(xmlText) + if resolvedKid == "" && kidHint != "" { + resolvedKid, _ = mpd.KIDToUUID(kidHint) + } + if (channel == "" || originBase == "") && resolvedKid != "" { + if mapped, base, ok := r.ResolveKID(mpd.KIDHex(resolvedKid)); ok { + if channel == "" { + channel = mapped + } + if originBase == "" { + originBase = base + } + } + } + if channel == "" { + channel = fallbackChannel + } + if originBase == "" { + originBase = fallbackBase + } + if channel != "" && originBase == "" { + originBase = cfg.Origins[channel] + } + if channel != "" && originBase != "" && r.usesEncoderTimeline(channel) { + if resolvedKid == "" { + resolvedKid = cfg.KIDs[channel] + } + if resolvedKid == "" { + return nil, "", "", fmt.Errorf("%s: no KID for synthetic MPD channel %s", Name, channel) + } + payload, err = mpd.BuildSynthetic(cfg.Synthetic, channel, originBase, resolvedKid) + return payload, channel, originBase, err + } + return nil, channel, originBase, fmt.Errorf("%s rewrite: unsupported MPD (no encoder channel); channel=%q", Name, channel) +} + +func (r *Rewriter) usesEncoderTimeline(channel string) bool { + ok, _ := regexp.MatchString(r.cfg.EncoderChannelRE, channel) + return ok +} + +// originFromXML finds the origin BaseURL and derives the encoder channel from it. +func (r *Rewriter) originFromXML(xmlText string) (channel, base string) { + marker := r.cfg.OriginMarker + if marker == "" { + marker = ".isml" + } + for _, m := range reBaseURL.FindAllStringSubmatch(xmlText, -1) { + text := strings.TrimSpace(m[1]) + idx := strings.Index(text, marker) + if idx < 0 { + continue + } + base = text[:idx] + marker + "/" + parts := strings.Split(strings.TrimSuffix(base, "/"), "/") + channel = strings.TrimSuffix(parts[len(parts)-1], marker) + if channel != "" { + return channel, base + } + } + return "", "" +} diff --git a/apps/modules/rte/rte.go b/apps/modules/rte/rte.go new file mode 100644 index 0000000..04e1326 --- /dev/null +++ b/apps/modules/rte/rte.go @@ -0,0 +1,118 @@ +// Package rte is the RTE Player app module: DASH + ModularDrm Widevine, captured +// through the phone MITM. Launch/auto-play are Go; every value (package name, +// license URL, channel chips, origins, KIDs) comes from the local, untracked +// apps/modules/rte/module.yaml, a flag, or the environment. +package rte + +import ( + "flag" + "fmt" + "time" + + "drmdecryption/adb" + appreg "drmdecryption/app" + "drmdecryption/capture" + "drmdecryption/modules/provider" + "drmdecryption/mpd" + "drmdecryption/uiflow" +) + +func init() { + appreg.Register(Name, New) + appreg.RegisterFlags(bindFlags) +} + +func bindFlags(fs *flag.FlagSet) { + fs.StringVar(&flags.packageName, Name+".package", "", "override "+Name+" android package id") + fs.StringVar(&flags.licenseURL, Name+".license-url", "", "override "+Name+" Widevine license URL") + fs.StringVar(&flags.caHash, Name+".ca-hash", "", "override "+Name+" MITM CA subject hash") + fs.StringVar(&flags.proxyBin, Name+".proxy-bin", "", "override "+Name+" on-device MITM binary") +} + +// App is the RTE app plugin. +type App struct { + *provider.Base + cfg Config +} + +// New constructs the plugin, loading values from module.yaml + flags + env. +func New() (appreg.App, error) { + cfg, res, err := loadConfig() + if err != nil { + return nil, err + } + labels := make(map[string]string, len(cfg.Channels)) + for id, ch := range cfg.Channels { + labels[id] = ch.Label + } + return &App{Base: provider.New(Name, cfg.Common, labels, res), cfg: cfg}, nil +} + +// KeyMode: RTE serves a thePlatform ModularDrm JSON license. +func (a *App) KeyMode() string { return "modulardrm" } + +// CaptureHints: the MITM must yield the auth token, program id, PSSH and manifest. +func (a *App) CaptureHints() capture.Hints { + return a.Hints("auth", "pid", "pssh", "mpd") +} + +// MPDRewriter publishes the synthetic encoder-timeline manifest. +func (a *App) MPDRewriter() mpd.Rewriter { return NewRewriterWith(a.cfg) } + +// Channels lists the configured live channels. +func (a *App) Channels() []appreg.Channel { + out := make([]appreg.Channel, 0, len(a.cfg.Channels)) + for _, id := range a.ChannelIDs() { + out = append(out, appreg.Channel{ID: id, Label: a.Label(id)}) + } + return out +} + +// Launch cold-starts the app and waits for its home UI. +func (a *App) Launch(c *adb.Client) error { + c.EnsureAwake() + fmt.Printf("[*] Launching %s (%s)…\n", Name, a.Package()) + c.ForceStop(a.Package()) + time.Sleep(400 * time.Millisecond) + if err := uiflow.MonkeyLaunch(c, a.Package()); err != nil { + return err + } + time.Sleep(time.Duration(a.cfg.Timeouts.LaunchSettle)) + c.DismissShadeIfFocused() + return uiflow.WaitUI(c, a.CacheDir(), uiflow.Match{ + DescContains: a.cfg.UI.LaunchDescContains, + }, time.Duration(a.cfg.Timeouts.LaunchWaitUI)) +} + +// AutoPlay navigates to a live channel and waits for playback. +func (a *App) AutoPlay(c *adb.Client, channel string) error { + id, err := a.Resolve(channel) + if err != nil { + return err + } + ch := a.cfg.Channels[id] + fmt.Printf("[*] Auto-play %s…\n", a.Label(id)) + c.DismissShadeIfFocused() + + if err := uiflow.TapUI(c, a.CacheDir(), uiflow.Match{ + DescRE: a.cfg.UI.LiveTabDescRE, + }, time.Duration(a.cfg.Timeouts.LiveTab)); err != nil { + return fmt.Errorf("live tab: %w", err) + } + time.Sleep(time.Duration(a.cfg.Timeouts.AfterLiveTab)) + + if err := a.tapChipThenPlay(c, ch); err != nil { + return err + } + return uiflow.WaitPlayback(c, a.CacheDir(), a.Package(), uiflow.PlaybackOpts{ + Timeout: time.Duration(a.cfg.Timeouts.Playback), + }) +} + +// StreamDefaults — the synthetic MPD carries a single 1080p + AAC pair, so a +// resolution filter is right here, and the rewriter must run. +func (a *App) VideoSelect() string { return "res=1280x720:for=best" } +func (a *App) AudioSelect() string { return "lang=en:for=best" } +func (a *App) RewriterName() string { return Name } +func (a *App) LiveWaitSeconds() int { return 2 } +func (a *App) TSReadyBytes() int64 { return 256 << 10 } diff --git a/apps/modules/rte/rte_test.go b/apps/modules/rte/rte_test.go new file mode 100644 index 0000000..eb08cc1 --- /dev/null +++ b/apps/modules/rte/rte_test.go @@ -0,0 +1,84 @@ +package rte + +import ( + "strings" + "testing" + + appreg "drmdecryption/app" + "drmdecryption/mpd" +) + +// Compile-time proof the plugin satisfies every interface the host expects. +var ( + _ appreg.App = (*App)(nil) + _ appreg.StreamDefaults = (*App)(nil) + _ mpd.Rewriter = (*Rewriter)(nil) + _ mpd.KIDResolver = (*Rewriter)(nil) + _ mpd.Primer = (*Rewriter)(nil) +) + +func testConfig() Config { + return Config{ + Channels: map[string]Channel{ + "chanone": {Label: "Chan One", ChipDesc: []string{"^chanone$"}}, + "chantwo": {Label: "Chan Two"}, + }, + Origins: map[string]string{ + "vc11": "https://origin.test/live/tc-1/vc11/vc11.isml/", + "vc12": "https://origin.test/live/tc-1/vc12/vc12.isml/", + }, + KIDs: map[string]string{ + "vc11": "df163382-1ddd-fdd5-bec9-822c1ec0f052", + }, + }.withDefaults() +} + +func TestResolveKIDMapsBackToOrigin(t *testing.T) { + r := NewRewriterWith(testConfig()) + ch, base, ok := r.ResolveKID("df1633821dddfdd5bec9822c1ec0f052") + if !ok { + t.Fatal("KID should resolve to a channel") + } + if ch != "vc11" { + t.Errorf("channel = %q, want vc11", ch) + } + if !strings.Contains(base, "vc11.isml") { + t.Errorf("origin base = %q", base) + } + if _, _, ok := r.ResolveKID("00000000000000000000000000000000"); ok { + t.Error("unknown KID must not resolve") + } +} + +func TestOriginFromXMLUsesConfiguredMarker(t *testing.T) { + r := NewRewriterWith(testConfig()) + xml := `https://origin.test/live/tc-1/vc12/vc12.isml/dash/` + ch, base := r.originFromXML(xml) + if ch != "vc12" { + t.Errorf("channel = %q, want vc12", ch) + } + if base != "https://origin.test/live/tc-1/vc12/vc12.isml/" { + t.Errorf("base = %q", base) + } +} + +// With no channel identifiable at all, the rewrite must fail rather than emit a +// manifest pointing nowhere. +func TestRewriteWithoutChannelFails(t *testing.T) { + r := NewRewriterWith(testConfig()) + if _, err := r.Rewrite([]byte(``), ""); err == nil { + t.Fatal("expected an error with no channel and no KID") + } +} + +// No provider identity may be compiled in: an empty config must yield no origins +// and no KIDs, so a fresh clone cannot stream without its local values file. +func TestNoHardcodedOrigins(t *testing.T) { + cfg := Config{}.withDefaults() + if len(cfg.Origins) != 0 || len(cfg.KIDs) != 0 { + t.Fatal("origins/KIDs must come from module.yaml, not from code") + } + if cfg.Package != "" || cfg.LicenseURL != "" { + t.Fatal("package/license URL must not be defaulted in code") + } +} diff --git a/apps/modules/rte/steps.go b/apps/modules/rte/steps.go new file mode 100644 index 0000000..4b92f0b --- /dev/null +++ b/apps/modules/rte/steps.go @@ -0,0 +1,61 @@ +package rte + +import ( + "fmt" + "regexp" + "time" + + "drmdecryption/adb" + "drmdecryption/uiflow" +) + +// tapChipThenPlay is this app's navigation idiom: on the Live tab, tap the +// channel chip, then the large hero Play button that appears. If the hero button +// is already on screen, tap it directly. +func (a *App) tapChipThenPlay(c *adb.Client, ch Channel) error { + playPats := uiflow.CompileRes(ch.PlayDesc) + chipPats := uiflow.CompileRes(ch.ChipDesc) + + nodes, err := c.DumpUI(a.CacheDir()) + if err != nil { + return err + } + if cand := a.findHeroPlay(nodes, playPats); cand != nil { + fmt.Printf("[*] tap Play @ %d,%d\n", cand.CX(), cand.CY()) + return c.Tap(cand.CX(), cand.CY()) + } + + chip := adb.FindSmallest(nodes, chipPats) + if chip == nil { + chip, err = c.WaitFor(a.CacheDir(), nil, chipPats, time.Duration(a.cfg.Timeouts.ChipWait)) + if err != nil { + return fmt.Errorf("chip not found: %w", err) + } + } + fmt.Printf("[*] tap chip @ %d,%d\n", chip.CX(), chip.CY()) + _ = c.Tap(chip.CX(), chip.CY()) + time.Sleep(time.Duration(a.cfg.Timeouts.AfterChip)) + + deadline := time.Now().Add(time.Duration(a.cfg.Timeouts.PlayWait)) + for time.Now().Before(deadline) { + nodes, err = c.DumpUI(a.CacheDir()) + if err == nil { + if cand := a.findHeroPlay(nodes, playPats); cand != nil { + fmt.Printf("[*] tap Play @ %d,%d\n", cand.CX(), cand.CY()) + return c.Tap(cand.CX(), cand.CY()) + } + } + time.Sleep(700 * time.Millisecond) + } + return fmt.Errorf("Play button did not appear") +} + +// findHeroPlay accepts a Play match only if it is big enough to be the hero +// button rather than a small list-item play icon. +func (a *App) findHeroPlay(nodes []adb.Node, playPats []*regexp.Regexp) *adb.Node { + cand := adb.FindLargest(nodes, nil, playPats) + if cand == nil || cand.Area() < a.cfg.HeroMinArea { + return nil + } + return cand +} diff --git a/apps/modules/tg4/catalog.go b/apps/modules/tg4/catalog.go new file mode 100644 index 0000000..a80501d --- /dev/null +++ b/apps/modules/tg4/catalog.go @@ -0,0 +1,151 @@ +package tg4 + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + appreg "drmdecryption/app" + "drmdecryption/brightcove" + "drmdecryption/session" +) + +// playbackConfig is the app's remote config blob. It is decoded generically: the +// field names a channel uses come from module.yaml (token_field / stream_id_field), +// so a renamed or added field needs no code change. +type playbackConfig map[string]any + +func fetchPlaybackConfig(url string) (playbackConfig, error) { + if url == "" { + return nil, fmt.Errorf("playback_config_url required (set it in module.yaml or --%s.playback-config-url)", Name) + } + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("playback config HTTP %d", resp.StatusCode) + } + var pc playbackConfig + if err := json.NewDecoder(resp.Body).Decode(&pc); err != nil { + return nil, err + } + return pc, nil +} + +// str reads a string field, tolerating numbers. +func (pc playbackConfig) str(field string) string { + if field == "" { + return "" + } + switch v := pc[field].(type) { + case string: + return v + case float64: + return fmt.Sprintf("%.0f", v) + case json.Number: + return v.String() + default: + return "" + } +} + +// Flags reports the boolean switches in the playback config, for `--list`. +func (pc playbackConfig) Flags() map[string]bool { + out := map[string]bool{} + for k, v := range pc { + if b, ok := v.(bool); ok { + out[k] = b + } + } + return out +} + +// resolveChannel returns the channel's label, Brightcove video id and playback +// token, consulting the remote playback config for anything not pinned locally. +func (a *App) resolveChannel(pc playbackConfig, id string) (label, videoID, token string, err error) { + ch, ok := a.cfg.Channels[id] + if !ok { + return "", "", "", fmt.Errorf("unknown channel %q", id) + } + label = a.Label(id) + videoID = ch.VideoID + if videoID == "" { + videoID = pc.str(ch.StreamIDField) + } + token = pc.str(ch.TokenField) + + if videoID == "" { + if ch.PhoneOnly { + return "", "", "", fmt.Errorf("channel %q has no catalog entry — capture it from the phone: drm capture --app %s --channel %s", id, Name, id) + } + return "", "", "", fmt.Errorf("no video id for channel %q (set video_id or stream_id_field in %s)", id, a.ConfigPath()) + } + if token == "" && ch.TokenField != "" { + return label, videoID, "", fmt.Errorf("no playback token for channel %q (field %s)", id, ch.TokenField) + } + return label, videoID, token, nil +} + +// Resolve implements app.Catalog: build streamd-ready credentials for a channel +// without touching a phone. +func (a *App) Resolve(channel string) (session.Stream, error) { + id, err := a.Base.Resolve(channel) + if err != nil { + return session.Stream{}, err + } + pc, err := fetchPlaybackConfig(a.cfg.PlaybackConfigURL) + if err != nil { + return session.Stream{}, err + } + label, videoID, token, err := a.resolveChannel(pc, id) + if err != nil { + return session.Stream{}, err + } + vid, err := brightcove.FetchPlayback(a.cfg.AccountID, videoID, token, a.cfg.PolicyKey) + if err != nil { + return session.Stream{}, err + } + hls, license := brightcove.PickHLSAndLicense(vid) + if hls == "" { + return session.Stream{}, fmt.Errorf("no HLS source in playback response for %s", videoID) + } + title := label + if vid.Name != "" { + title = vid.Name + } + return session.Stream{ + Name: a.cfg.StreamNamePrefix + "-" + strings.ToLower(id), + Title: title, + App: Name, + Channel: strings.ToLower(id), + MPD: hls, + HeadersJSON: "{}", + Rewriter: "none", + LicenseURL: license, + AccountID: a.cfg.AccountID, + VideoID: videoID, + PlaybackURL: brightcove.PlaybackURL(a.cfg.AccountID, videoID), + ManifestType: "hls", + }, nil +} + +// ListChannels resolves each configured channel's video id for `catalog --list`. +func (a *App) ListChannels() ([]appreg.CatalogRow, map[string]bool, error) { + pc, err := fetchPlaybackConfig(a.cfg.PlaybackConfigURL) + if err != nil { + return nil, nil, err + } + rows := make([]appreg.CatalogRow, 0, len(a.cfg.Channels)) + for _, id := range a.ChannelIDs() { + ch := a.cfg.Channels[id] + vid := ch.VideoID + if vid == "" { + vid = pc.str(ch.StreamIDField) + } + rows = append(rows, appreg.CatalogRow{ID: id, CatalogID: vid, Label: a.Label(id)}) + } + return rows, pc.Flags(), nil +} diff --git a/apps/modules/tg4/config.go b/apps/modules/tg4/config.go new file mode 100644 index 0000000..2162ed7 --- /dev/null +++ b/apps/modules/tg4/config.go @@ -0,0 +1,172 @@ +package tg4 + +import ( + "time" + + "drmdecryption/modcfg" + "drmdecryption/modules/provider" + "drmdecryption/uiflow" +) + +// Name is the module id (--app tg4). +const Name = "tg4" + +// Channel is one live channel. VideoID may be empty when the id must be read from +// the remote playback config instead. +type Channel struct { + Label string `yaml:"label"` + // VideoID is the Brightcove video id for this live. + VideoID string `yaml:"video_id"` + // TokenField names the playback-config field holding this live's playback token. + TokenField string `yaml:"token_field"` + // StreamIDField names the playback-config field holding this live's video id, + // used when VideoID is empty. + StreamIDField string `yaml:"stream_id_field"` + // LiveCardIndex is this channel's position in the app's Live card list. + LiveCardIndex int `yaml:"live_card_index"` + // PhoneOnly marks a channel with no catalog entry — capture via the phone UI. + PhoneOnly bool `yaml:"phone_only"` +} + +// Config is apps/modules/tg4/module.yaml. +type Config struct { + provider.Common `yaml:",inline"` + + // Brightcove account credentials. Secrets: never defaulted in code. + AccountID string `yaml:"account_id"` + PolicyKey string `yaml:"policy_key"` + PlaybackConfigURL string `yaml:"playback_config_url"` + + Channels map[string]Channel `yaml:"channels"` + + // StreamNamePrefix prefixes generated stream names (default: module name). + StreamNamePrefix string `yaml:"stream_name_prefix"` + + Cards uiflow.CardOpts `yaml:"cards"` + UI UI `yaml:"ui"` + Timeouts Timeouts `yaml:"timeouts"` +} + +// UI holds the markers and fallbacks the Live-page navigation uses. +type UI struct { + // LaunchDescContains: any of these content-descriptions means the app is up. + LaunchDescContains []string `yaml:"launch_desc_contains"` + // LaunchResourceContains: any of these view ids means the app is up. More + // stable than descriptions, which are localized. + LaunchResourceContains []string `yaml:"launch_resource_contains"` + // LiveLabel is the menu/title text identifying the Live page. + LiveLabel string `yaml:"live_label"` + // LiveTitleResource marks the Live page title node. + LiveTitleResource string `yaml:"live_title_resource"` + // DrawerDesc are the content-descriptions of the drawer (hamburger) button. + DrawerDesc []string `yaml:"drawer_desc"` + // DrawerFallbackX/Y is tapped when the drawer button cannot be identified. + DrawerFallbackX int `yaml:"drawer_fallback_x"` + DrawerFallbackY int `yaml:"drawer_fallback_y"` + // MenuMaxX / MenuMinY / MenuMaxY bound where a drawer menu entry can sit. + MenuMaxX int `yaml:"menu_max_x"` + MenuMinY int `yaml:"menu_min_y"` + MenuMaxY int `yaml:"menu_max_y"` + // TitleMaxY bounds where the Live page title can sit. + TitleMaxY int `yaml:"title_max_y"` + // PlaybackDescContains / PlaybackResourceContains are player-up signals. + PlaybackDescContains []string `yaml:"playback_desc_contains"` + PlaybackResourceContains []string `yaml:"playback_resource_contains"` +} + +// Timeouts for the launch / autoplay sequence. Written as "25s" in module.yaml. +type Timeouts struct { + LaunchWaitUI provider.Duration `yaml:"launch_wait_ui"` + LaunchSettle provider.Duration `yaml:"launch_settle"` + LiveNav provider.Duration `yaml:"live_nav"` + Card provider.Duration `yaml:"card"` + AfterCard provider.Duration `yaml:"after_card"` + Playback provider.Duration `yaml:"playback"` + AfterLiveTap provider.Duration `yaml:"after_live_tap"` + AfterDrawer provider.Duration `yaml:"after_drawer"` +} + +func (c Config) withDefaults() Config { + if c.StreamNamePrefix == "" { + c.StreamNamePrefix = Name + } + c.Cards = c.Cards.WithDefaults() + if len(c.Cards.DescDeny) == 0 { + c.Cards.DescDeny = []string{"Search", "Profile", "Cast", "Open", "Closed"} + } + if len(c.Cards.TextDeny) == 0 { + c.Cards.TextDeny = []string{"catch-up"} + } + u := &c.UI + if len(u.LaunchDescContains) == 0 && len(u.LaunchResourceContains) == 0 { + u.LaunchDescContains = []string{"live", "home"} + } + if u.LiveLabel == "" { + u.LiveLabel = "live" + } + if u.LiveTitleResource == "" { + u.LiveTitleResource = "live_fragment_text_view" + } + if len(u.DrawerDesc) == 0 { + u.DrawerDesc = []string{"Open", "Closed"} + } + if u.DrawerFallbackX == 0 { + u.DrawerFallbackX = 68 + } + if u.DrawerFallbackY == 0 { + u.DrawerFallbackY = 183 + } + if u.MenuMaxX == 0 { + u.MenuMaxX = 700 + } + if u.MenuMinY == 0 { + u.MenuMinY = 250 + } + if u.MenuMaxY == 0 { + u.MenuMaxY = 700 + } + if u.TitleMaxY == 0 { + u.TitleMaxY = 400 + } + if len(u.PlaybackDescContains) == 0 { + u.PlaybackDescContains = []string{"video player"} + } + if len(u.PlaybackResourceContains) == 0 { + u.PlaybackResourceContains = []string{"brightcove_video_view"} + } + t := &c.Timeouts + t.LaunchWaitUI = provider.Duration(t.LaunchWaitUI.D(30 * time.Second)) + t.LaunchSettle = provider.Duration(t.LaunchSettle.D(5 * time.Second)) + t.LiveNav = provider.Duration(t.LiveNav.D(25 * time.Second)) + t.Card = provider.Duration(t.Card.D(25 * time.Second)) + t.AfterCard = provider.Duration(t.AfterCard.D(3 * time.Second)) + t.Playback = provider.Duration(t.Playback.D(60 * time.Second)) + t.AfterLiveTap = provider.Duration(t.AfterLiveTap.D(3 * time.Second)) + t.AfterDrawer = provider.Duration(t.AfterDrawer.D(1200 * time.Millisecond)) + return c +} + +// flag overrides, bound before any app is constructed. +var flags struct { + packageName string + accountID string + policyKey string + playbackConfigURL string + caHash string + proxyBin string +} + +func loadConfig() (Config, modcfg.Result, error) { + var cfg Config + res, err := modcfg.Load(Name, &cfg) + if err != nil { + return cfg, res, err + } + cfg.Package = modcfg.Override(Name, "package", flags.packageName, cfg.Package) + cfg.CAHash = modcfg.Override(Name, "ca_hash", flags.caHash, cfg.CAHash) + cfg.ProxyBin = modcfg.Override(Name, "proxy_bin", flags.proxyBin, cfg.ProxyBin) + cfg.AccountID = modcfg.Override(Name, "account_id", flags.accountID, cfg.AccountID) + cfg.PolicyKey = modcfg.Override(Name, "policy_key", flags.policyKey, cfg.PolicyKey) + cfg.PlaybackConfigURL = modcfg.Override(Name, "playback_config_url", flags.playbackConfigURL, cfg.PlaybackConfigURL) + return cfg.withDefaults(), res, nil +} diff --git a/apps/modules/tg4/keys.go b/apps/modules/tg4/keys.go new file mode 100644 index 0000000..a14db24 --- /dev/null +++ b/apps/modules/tg4/keys.go @@ -0,0 +1,62 @@ +package tg4 + +import ( + "fmt" + "path/filepath" + "strings" + + "drmdecryption/brightcove" + "drmdecryption/repo" + "drmdecryption/session" + "drmdecryption/wvkey" +) + +// EnrichWithKeys fetches the HLS master, extracts the Widevine PSSH and resolves +// every content key through the local CDM. Brightcove returns multiple CONTENT +// keys and a downloader needs all of them. +func (a *App) EnrichWithKeys(info *session.Stream, python, wvd string) error { + if info == nil || info.MPD == "" { + return fmt.Errorf("missing manifest URL") + } + if info.LicenseURL == "" { + return fmt.Errorf("missing license_url") + } + if strings.TrimSpace(wvd) == "" { + return fmt.Errorf("key fetch requires --wvd (path to .wvd device file)") + } + body, err := brightcove.Get(info.MPD) + if err != nil { + return fmt.Errorf("hls master: %w", err) + } + pssh := brightcove.ExtractWidevinePSSH(body) + if pssh == "" { + return fmt.Errorf("no Widevine PSSH in HLS master") + } + info.PSSH = pssh + info.PrimaryKID = brightcove.PrimaryKID(body) + + all, err := wvkey.FetchRawAll(wvkey.Options{ + Python: python, + Script: filepath.Join(repo.Root(), "apps", "wvkey", "wvkey.py"), + WVD: wvd, + PSSH: pssh, + LicenseURL: info.LicenseURL, + }) + if err != nil { + return err + } + if len(all) == 0 { + return fmt.Errorf("license returned no content keys") + } + info.Keys = all + info.Key = all[0] + if info.PrimaryKID != "" { + for _, k := range all { + if strings.HasPrefix(strings.ToLower(k), info.PrimaryKID+":") { + info.Key = k + break + } + } + } + return nil +} diff --git a/apps/modules/tg4/steps.go b/apps/modules/tg4/steps.go new file mode 100644 index 0000000..3b5040b --- /dev/null +++ b/apps/modules/tg4/steps.go @@ -0,0 +1,152 @@ +package tg4 + +import ( + "fmt" + "strings" + "time" + + "drmdecryption/adb" + "drmdecryption/uiflow" +) + +// openLivePage navigates to the Live page: if a menu entry is visible tap it, +// otherwise open the side drawer first. Ported from the Starlark hook this module +// used to carry, with every selector and coordinate now coming from config. +func (a *App) openLivePage(c *adb.Client) error { + c.DismissShadeIfFocused() + nodes, err := c.DumpUI(a.CacheDir()) + if err != nil { + return err + } + if a.onLivePage(nodes) { + fmt.Println("[*] already on Live") + return nil + } + if live := a.findMenuLive(nodes); live != nil { + fmt.Printf("[*] tap Live @ %d,%d\n", live.CX(), live.CY()) + _ = c.Tap(live.CX(), live.CY()) + time.Sleep(time.Duration(a.cfg.Timeouts.AfterLiveTap)) + return nil + } + + if !a.openDrawer(c) { + fmt.Printf("[*] drawer fallback tap %d,%d\n", a.cfg.UI.DrawerFallbackX, a.cfg.UI.DrawerFallbackY) + _ = c.Tap(a.cfg.UI.DrawerFallbackX, a.cfg.UI.DrawerFallbackY) + time.Sleep(time.Duration(a.cfg.Timeouts.AfterDrawer)) + } + + deadline := time.Now().Add(time.Duration(a.cfg.Timeouts.LiveNav)) + for time.Now().Before(deadline) { + nodes, err := c.DumpUI(a.CacheDir()) + if err == nil { + if live := a.findMenuLive(nodes); live != nil { + fmt.Printf("[*] tap Live @ %d,%d\n", live.CX(), live.CY()) + _ = c.Tap(live.CX(), live.CY()) + time.Sleep(time.Duration(a.cfg.Timeouts.AfterLiveTap)) + break + } + a.openDrawer(c) + } + time.Sleep(800 * time.Millisecond) + } + + deadline = time.Now().Add(time.Duration(a.cfg.Timeouts.LiveNav)) + for time.Now().Before(deadline) { + if nodes, err := c.DumpUI(a.CacheDir()); err == nil && a.onLivePage(nodes) { + fmt.Printf("[*] Live page ready\n") + return nil + } + time.Sleep(800 * time.Millisecond) + } + return fmt.Errorf("Live page did not open") +} + +// onLivePage is true when the Live title is showing and at least one content card +// is on screen. +func (a *App) onLivePage(nodes []adb.Node) bool { + hasTitle := false + for _, n := range nodes { + if !strings.EqualFold(strings.TrimSpace(n.Text), a.cfg.UI.LiveLabel) { + continue + } + if strings.Contains(n.ResourceID, a.cfg.UI.LiveTitleResource) || + (n.Y1 < a.cfg.UI.TitleMaxY && !n.Clickable) { + hasTitle = true + break + } + } + if !hasTitle { + return false + } + return len(uiflow.Cards(nodes, a.cfg.Cards)) > 0 +} + +// openDrawer opens the side menu. Returns false when the drawer button could not +// be identified, so the caller can fall back to a fixed tap. +func (a *App) openDrawer(c *adb.Client) bool { + nodes, err := c.DumpUI(a.CacheDir()) + if err != nil { + return false + } + // Already open? + for _, n := range nodes { + desc := strings.TrimSpace(n.Desc) + for _, d := range a.cfg.UI.DrawerDesc { + if desc == d && d == "Closed" { + return true + } + } + if strings.EqualFold(strings.TrimSpace(n.Text), a.cfg.UI.LiveLabel) && + n.Clickable && n.X2 < a.cfg.UI.MenuMaxX { + return true + } + } + ham := a.findDrawerButton(nodes) + if ham == nil { + return false + } + fmt.Printf("[*] tap menu %q @ %d,%d\n", ham.Desc, ham.CX(), ham.CY()) + _ = c.Tap(ham.CX(), ham.CY()) + time.Sleep(time.Duration(a.cfg.Timeouts.AfterDrawer)) + return true +} + +func (a *App) findDrawerButton(nodes []adb.Node) *adb.Node { + for i := range nodes { + n := &nodes[i] + if !n.Clickable { + continue + } + desc := strings.TrimSpace(n.Desc) + for _, d := range a.cfg.UI.DrawerDesc { + if desc == d { + return n + } + } + } + // Geometric fallback: a small clickable node in the top-left corner. + for i := range nodes { + n := &nodes[i] + if n.Clickable && n.X1 < 50 && n.Y1 < 200 && n.X2 < 200 && n.Y2 < 300 { + return n + } + } + return nil +} + +// findMenuLive locates the drawer's Live entry. +func (a *App) findMenuLive(nodes []adb.Node) *adb.Node { + for i := range nodes { + n := &nodes[i] + if !n.Clickable { + continue + } + if !strings.EqualFold(strings.TrimSpace(n.Text), a.cfg.UI.LiveLabel) { + continue + } + if n.X2 <= a.cfg.UI.MenuMaxX && n.Y1 > a.cfg.UI.MenuMinY && n.Y1 < a.cfg.UI.MenuMaxY { + return n + } + } + return nil +} diff --git a/apps/modules/tg4/tg4.go b/apps/modules/tg4/tg4.go new file mode 100644 index 0000000..8e20653 --- /dev/null +++ b/apps/modules/tg4/tg4.go @@ -0,0 +1,125 @@ +// Package tg4 is the TG4 app module: Brightcove HLS with a raw Widevine license. +// Its channels can be resolved from the public catalog (no phone), or captured +// through the phone MITM. Account id, policy key, video ids and the playback +// config URL all come from the local, untracked apps/modules/tg4/module.yaml, a +// flag, or the environment — never from this source. +package tg4 + +import ( + "flag" + "fmt" + "time" + + "drmdecryption/adb" + appreg "drmdecryption/app" + "drmdecryption/capture" + "drmdecryption/modules/provider" + "drmdecryption/mpd" + "drmdecryption/uiflow" +) + +func init() { + appreg.Register(Name, New) + appreg.RegisterFlags(bindFlags) +} + +func bindFlags(fs *flag.FlagSet) { + fs.StringVar(&flags.packageName, Name+".package", "", "override "+Name+" android package id") + fs.StringVar(&flags.accountID, Name+".account-id", "", "override "+Name+" Brightcove account id") + fs.StringVar(&flags.policyKey, Name+".policy-key", "", "override "+Name+" Brightcove policy key") + fs.StringVar(&flags.playbackConfigURL, Name+".playback-config-url", "", "override "+Name+" playback config URL") + fs.StringVar(&flags.caHash, Name+".ca-hash", "", "override "+Name+" MITM CA subject hash") + fs.StringVar(&flags.proxyBin, Name+".proxy-bin", "", "override "+Name+" on-device MITM binary") +} + +// App is the TG4 app plugin. +type App struct { + *provider.Base + cfg Config +} + +// New constructs the plugin, loading values from module.yaml + flags + env. +func New() (appreg.App, error) { + cfg, res, err := loadConfig() + if err != nil { + return nil, err + } + labels := make(map[string]string, len(cfg.Channels)) + for id, ch := range cfg.Channels { + labels[id] = ch.Label + } + return &App{Base: provider.New(Name, cfg.Common, labels, res), cfg: cfg}, nil +} + +// KeyMode: Brightcove serves a raw octet-stream Widevine license. +func (a *App) KeyMode() string { return "raw" } + +// CaptureHints: the MITM yields the license URL and HLS master; the PSSH is read +// from the master afterwards. +func (a *App) CaptureHints() capture.Hints { + return a.Hints("license_url", "mpd") +} + +// MPDRewriter: HLS needs no manifest rewrite. +func (a *App) MPDRewriter() mpd.Rewriter { return mpd.Passthrough{} } + +// Channels lists the configured live channels. +func (a *App) Channels() []appreg.Channel { + out := make([]appreg.Channel, 0, len(a.cfg.Channels)) + for _, id := range a.ChannelIDs() { + out = append(out, appreg.Channel{ID: id, Label: a.Label(id)}) + } + return out +} + +// Launch cold-starts the app and waits for its home UI. +func (a *App) Launch(c *adb.Client) error { + c.EnsureAwake() + fmt.Printf("[*] Launching %s (%s)…\n", Name, a.Package()) + c.ForceStop(a.Package()) + time.Sleep(400 * time.Millisecond) + if err := uiflow.MonkeyLaunch(c, a.Package()); err != nil { + return err + } + time.Sleep(time.Duration(a.cfg.Timeouts.LaunchSettle)) + c.DismissShadeIfFocused() + return uiflow.WaitUI(c, a.CacheDir(), uiflow.Match{ + DescContains: a.cfg.UI.LaunchDescContains, + ResourceContains: a.cfg.UI.LaunchResourceContains, + }, time.Duration(a.cfg.Timeouts.LaunchWaitUI)) +} + +// AutoPlay opens the Live page and taps this channel's card. +func (a *App) AutoPlay(c *adb.Client, channel string) error { + id, err := a.Base.Resolve(channel) + if err != nil { + return err + } + ch := a.cfg.Channels[id] + fmt.Printf("[*] Auto-play %s…\n", a.Label(id)) + + if err := a.openLivePage(c); err != nil { + return err + } + if err := uiflow.TapCard(c, a.CacheDir(), ch.LiveCardIndex, time.Duration(a.cfg.Timeouts.Card), a.cfg.Cards); err != nil { + return err + } + time.Sleep(time.Duration(a.cfg.Timeouts.AfterCard)) + + // Soft-fail: this player often starts without a dumpsys audio session, so a + // UI signal is the real confirmation and capture should continue regardless. + return uiflow.WaitPlayback(c, a.CacheDir(), a.Package(), uiflow.PlaybackOpts{ + Timeout: time.Duration(a.cfg.Timeouts.Playback), + SoftFail: true, + OrDescContains: a.cfg.UI.PlaybackDescContains, + OrResourceContains: a.cfg.UI.PlaybackResourceContains, + }) +} + +// StreamDefaults — multi-KID SAMPLE-AES HLS: take the best variant and give the +// muxer more bytes before probing, since early audio config is often incomplete. +func (a *App) VideoSelect() string { return "for=best" } +func (a *App) AudioSelect() string { return "for=best" } +func (a *App) RewriterName() string { return "none" } +func (a *App) LiveWaitSeconds() int { return 6 } +func (a *App) TSReadyBytes() int64 { return 2 << 20 } diff --git a/apps/modules/tg4/tg4_test.go b/apps/modules/tg4/tg4_test.go new file mode 100644 index 0000000..cc16fc6 --- /dev/null +++ b/apps/modules/tg4/tg4_test.go @@ -0,0 +1,129 @@ +package tg4 + +import ( + "testing" + + appreg "drmdecryption/app" + "drmdecryption/modcfg" + "drmdecryption/modules/provider" +) + +// Compile-time proof the plugin satisfies every interface the host expects. +var ( + _ appreg.App = (*App)(nil) + _ appreg.Catalog = (*App)(nil) + _ appreg.StreamDefaults = (*App)(nil) +) + +func testApp(t *testing.T) *App { + t.Helper() + cfg := Config{ + AccountID: "acct", + PolicyKey: "pk", + PlaybackConfigURL: "https://cfg.test/playback.json", + Channels: map[string]Channel{ + "main": {Label: "Main", VideoID: "111", TokenField: "mainToken", LiveCardIndex: 0}, + "extra": {Label: "Extra", StreamIDField: "extraStreamId", TokenField: "extraToken", LiveCardIndex: 1}, + "phone": {Label: "Phone only", PhoneOnly: true, LiveCardIndex: 3}, + }, + Common: provider.Common{ + Aliases: map[string]string{"primary": "main", "second": "extra"}, + }, + }.withDefaults() + labels := map[string]string{} + for id, ch := range cfg.Channels { + labels[id] = ch.Label + } + return &App{Base: provider.New(Name, cfg.Common, labels, modcfg.Result{Path: "test"}), cfg: cfg} +} + +func TestAliasesResolveFromConfig(t *testing.T) { + a := testApp(t) + for in, want := range map[string]string{ + "main": "main", "MAIN": "main", "primary": "main", + "second": "extra", "extra": "extra", + } { + got, err := a.Base.Resolve(in) + if err != nil { + t.Fatalf("resolve %q: %v", in, err) + } + if got != want { + t.Errorf("resolve(%q) = %q, want %q", in, got, want) + } + } + if _, err := a.Base.Resolve("nope"); err == nil { + t.Error("unknown channel must error") + } +} + +// A video id pinned in config wins; otherwise it is read from the named remote +// playback-config field. Both paths are data-driven, no channel name switch. +func TestResolveChannelReadsConfiguredFields(t *testing.T) { + a := testApp(t) + pc := playbackConfig{ + "mainToken": "tok-main", + "extraToken": "tok-extra", + "extraStreamId": "222", + } + _, vid, tok, err := a.resolveChannel(pc, "main") + if err != nil { + t.Fatalf("main: %v", err) + } + if vid != "111" || tok != "tok-main" { + t.Errorf("main → video %q token %q", vid, tok) + } + _, vid, tok, err = a.resolveChannel(pc, "extra") + if err != nil { + t.Fatalf("extra: %v", err) + } + if vid != "222" || tok != "tok-extra" { + t.Errorf("extra → video %q token %q", vid, tok) + } +} + +// A phone-only channel must say so rather than fail with a confusing catalog error. +func TestPhoneOnlyChannelExplainsItself(t *testing.T) { + a := testApp(t) + _, _, _, err := a.resolveChannel(playbackConfig{}, "phone") + if err == nil { + t.Fatal("phone-only channel must not resolve from the catalog") + } + if want := "drm capture"; !contains(err.Error(), want) { + t.Errorf("error %q should point at %q", err, want) + } +} + +func TestPlaybackConfigCoercesNumericIDs(t *testing.T) { + pc := playbackConfig{"a": "123", "b": float64(456), "c": true} + if pc.str("a") != "123" { + t.Error("string field") + } + if pc.str("b") != "456" { + t.Errorf("numeric field = %q, want 456", pc.str("b")) + } + if pc.str("missing") != "" || pc.str("") != "" { + t.Error("missing field must be empty") + } + if !pc.Flags()["c"] { + t.Error("bool flags should be reported") + } +} + +// No provider identity may be compiled in. +func TestNoHardcodedCredentials(t *testing.T) { + cfg := Config{}.withDefaults() + if cfg.AccountID != "" || cfg.PolicyKey != "" || cfg.PlaybackConfigURL != "" || cfg.Package != "" { + t.Fatal("account id / policy key / config URL / package must come from module.yaml") + } +} + +func contains(s, sub string) bool { + return len(s) >= len(sub) && (func() bool { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return true + } + } + return false + })() +} diff --git a/apps/pkg/README.md b/apps/pkg/README.md new file mode 100644 index 0000000..05a5413 --- /dev/null +++ b/apps/pkg/README.md @@ -0,0 +1,37 @@ +# pkg — shared Go libraries + +Imported by `apps/capture`, `apps/agent`, `apps/streamd`, `apps/proxy`, `apps/cli` +and `apps/modules` as module `drmdecryption` via: + +```go +replace drmdecryption => ../pkg +``` + +## Packages + +| Package | Role | +|---------|------| +| `adb` | ADB client (shell, push/pull, UI dump, tap) | +| `app` | App plugin interface + registry (+ optional Catalog / StreamDefaults) | +| `uiflow` | Phone-UI primitives app modules drive playback with | +| `modcfg` | Load an app module's local `module.yaml` + env overrides | +| `proxy` | Push/start/stop appproxy, CA install/reinject, pull captures | +| `phonecap` | One phone MITM job (proxy → launch → autoplay → wvkey → session) | +| `wvkey` | Shells out to `apps/wvkey/wvkey.py` | +| `brightcove` | Brightcove Playback API + HLS Widevine extraction (platform, not provider) | +| `capture` | Parse on-device capture JSON + score candidate manifest URLs | +| `session` | Write `outputs///` | +| `mpd` | Manifest rewriter interface, registry, synthetic live MPD builder, local server | +| `repo` | Find repo root (`apps/pkg/go.mod`) | + +Nothing here names a streaming provider. Provider logic lives in `apps/modules/`; +provider values live in that module's gitignored `module.yaml`. + +## Build / test + +```bash +# Go must be on PATH (or set GOROOT) +go -C apps/pkg test ./... +``` + +No binary of its own — everything ships in `bin/drm`, built from `apps/cli`. diff --git a/apps/pkg/adb/adb.go b/apps/pkg/adb/adb.go new file mode 100644 index 0000000..eb1fa3a --- /dev/null +++ b/apps/pkg/adb/adb.go @@ -0,0 +1,374 @@ +package adb + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "time" +) + +// Client wraps the adb CLI. When Serial is set, every invocation uses -s SERIAL. +type Client struct { + Bin string + Serial string +} + +func New() *Client { + bin := "adb" + if p, err := exec.LookPath("adb"); err == nil { + bin = p + } + return &Client{Bin: bin} +} + +// WithSerial returns a copy that targets one device. +func (c *Client) WithSerial(serial string) *Client { + out := *c + out.Serial = strings.TrimSpace(serial) + return &out +} + +func (c *Client) prefix(args []string) []string { + if c.Serial == "" { + return args + } + return append([]string{"-s", c.Serial}, args...) +} + +func (c *Client) Run(args ...string) (string, string, error) { + cmd := exec.Command(c.Bin, c.prefix(args)...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return stdout.String(), stderr.String(), err +} + +func (c *Client) Out(args ...string) string { + out, _, _ := c.Run(args...) + return strings.TrimSpace(out) +} + +func (c *Client) EnsureDevice() error { + state := c.Out("get-state") + if state != "device" { + if c.Serial != "" { + return fmt.Errorf("adb device %s not ready (state=%q)", c.Serial, state) + } + return fmt.Errorf("adb device not ready (state=%q)", state) + } + return nil +} + +// Device is one row from `adb devices -l`. +type Device struct { + Serial string + State string + Model string + Product string + USB string +} + +// ListDevices returns every adb device row (any state). Uses a serial-less client. +func ListDevices(bin string) ([]Device, error) { + if bin == "" { + bin = New().Bin + } + cmd := exec.Command(bin, "devices", "-l") + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("adb devices: %w (%s)", err, strings.TrimSpace(stderr.String())) + } + var out []Device + for _, line := range strings.Split(stdout.String(), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "List of devices") { + continue + } + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + d := Device{Serial: fields[0], State: fields[1]} + for _, f := range fields[2:] { + if strings.HasPrefix(f, "model:") { + d.Model = strings.TrimPrefix(f, "model:") + } + if strings.HasPrefix(f, "product:") { + d.Product = strings.TrimPrefix(f, "product:") + } + if strings.HasPrefix(f, "usb:") { + d.USB = strings.TrimPrefix(f, "usb:") + } + } + out = append(out, d) + } + return out, nil +} + +// PackageInstalled is true when `pm path ` returns a path. Never installs. +func (c *Client) PackageInstalled(pkg string) bool { + pkg = strings.TrimSpace(pkg) + if pkg == "" { + return false + } + out := c.Out("shell", "pm", "path", pkg) + return strings.Contains(out, "package:") +} + +// ForceStop stops the package so the phone is free for the next job. +func (c *Client) ForceStop(pkg string) { + if pkg == "" { + return + } + _, _ = c.Shell("am", "force-stop", pkg) +} + +// EnsureAwake wakes the screen. Call this BEFORE launching the target app. +// It must not press HOME or swipe after the app is open — dumpsys always +// contains "StatusBar", and a HOME key looks like "the app just exits". +func (c *Client) EnsureAwake() { + _, _ = c.Shell("input", "keyevent", "KEYCODE_WAKEUP") + time.Sleep(250 * time.Millisecond) + _, _ = c.Shell("wm", "dismiss-keyguard") + + if currentFocusIsLockOrShade(c.Out("shell", "dumpsys", "window")) { + _, _ = c.Shell("input", "swipe", "540", "2000", "540", "600", "300") + time.Sleep(400 * time.Millisecond) + c.DismissShadeIfFocused() + } +} + +func currentFocusIsLockOrShade(dumpsys string) bool { + for _, line := range strings.Split(dumpsys, "\n") { + line = strings.TrimSpace(line) + if !strings.Contains(line, "mCurrentFocus=") { + continue + } + low := strings.ToLower(line) + return strings.Contains(low, "notificationshade") || + strings.Contains(low, "keyguard") || + strings.Contains(low, "lockscreen") + } + return false +} + +// DismissShadeIfFocused presses BACK only when the notification shade has focus. +func (c *Client) DismissShadeIfFocused() { + if currentFocusIsLockOrShade(c.Out("shell", "dumpsys", "window")) { + _, _ = c.Shell("input", "keyevent", "KEYCODE_BACK") + time.Sleep(200 * time.Millisecond) + } +} + +func (c *Client) Shell(args ...string) (string, error) { + all := append([]string{"shell"}, args...) + out, errOut, err := c.Run(all...) + if err != nil { + if strings.TrimSpace(errOut) != "" { + return out, fmt.Errorf("%w: %s", err, strings.TrimSpace(errOut)) + } + return out, err + } + return out, nil +} + +func (c *Client) Push(local, remote string) error { + _, errOut, err := c.Run("push", local, remote) + if err != nil { + return fmt.Errorf("adb push: %w (%s)", err, strings.TrimSpace(errOut)) + } + return nil +} + +func (c *Client) Pull(remote, local string) error { + _, errOut, err := c.Run("pull", remote, local) + if err != nil { + return fmt.Errorf("adb pull: %w (%s)", err, strings.TrimSpace(errOut)) + } + return nil +} + +func (c *Client) Tap(x, y int) error { + _, err := c.Shell("input", "tap", strconv.Itoa(x), strconv.Itoa(y)) + return err +} + +// Swipe performs an input swipe over durationMs milliseconds. +func (c *Client) Swipe(x1, y1, x2, y2, durationMs int) error { + if durationMs <= 0 { + durationMs = 300 + } + _, err := c.Shell("input", "swipe", + strconv.Itoa(x1), strconv.Itoa(y1), + strconv.Itoa(x2), strconv.Itoa(y2), + strconv.Itoa(durationMs)) + return err +} + +// Node is one uiautomator element. +type Node struct { + Text string + Desc string + Class string + ResourceID string + Clickable bool + X1, Y1 int + X2, Y2 int +} + +func (n Node) CX() int { return (n.X1 + n.X2) / 2 } +func (n Node) CY() int { return (n.Y1 + n.Y2) / 2 } +func (n Node) Area() int { + return (n.X2 - n.X1) * (n.Y2 - n.Y1) +} + +var ( + reNode = regexp.MustCompile(`]+>`) + reAttr = func(k string) *regexp.Regexp { return regexp.MustCompile(k + `="([^"]*)"`) } + reBounds = regexp.MustCompile(`\[(\d+),(\d+)\]\[(\d+),(\d+)\]`) +) + +func parseNodes(xml string) []Node { + var out []Node + for _, m := range reNode.FindAllString(xml, -1) { + attr := func(k string) string { + mm := reAttr(k).FindStringSubmatch(m) + if len(mm) < 2 { + return "" + } + return mm[1] + } + b := attr("bounds") + bm := reBounds.FindStringSubmatch(b) + if len(bm) != 5 { + continue + } + x1, _ := strconv.Atoi(bm[1]) + y1, _ := strconv.Atoi(bm[2]) + x2, _ := strconv.Atoi(bm[3]) + y2, _ := strconv.Atoi(bm[4]) + out = append(out, Node{ + Text: attr("text"), + Desc: attr("content-desc"), + Class: attr("class"), + ResourceID: attr("resource-id"), + Clickable: attr("clickable") == "true", + X1: x1, Y1: y1, X2: x2, Y2: y2, + }) + } + return out +} + +// DumpUI pulls a uiautomator hierarchy. +func (c *Client) DumpUI(cacheDir string) ([]Node, error) { + remote := "/sdcard/streamd_uidump.xml" + if _, err := c.Shell("uiautomator", "dump", remote); err != nil { + // dump often returns exit 0 with message; ignore soft failures + } + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return nil, err + } + local := cacheDir + string(os.PathSeparator) + "uidump.xml" + if err := c.Pull(remote, local); err != nil { + return nil, err + } + raw, err := os.ReadFile(local) + if err != nil { + return nil, err + } + return parseNodes(string(raw)), nil +} + +// FindLargest returns the largest node matching any text/desc regex. +func FindLargest(nodes []Node, textPats, descPats []*regexp.Regexp) *Node { + var hits []Node + for _, n := range nodes { + ok := false + if n.Text != "" { + for _, p := range textPats { + if p.MatchString(n.Text) { + ok = true + break + } + } + } + if n.Desc != "" { + for _, p := range descPats { + if p.MatchString(n.Desc) { + ok = true + break + } + } + } + if ok { + hits = append(hits, n) + } + } + if len(hits) == 0 { + return nil + } + best := hits[0] + for _, h := range hits[1:] { + if h.Area() > best.Area() { + best = h + } + } + return &best +} + +// FindSmallest prefers tiny chips over hero buttons. +func FindSmallest(nodes []Node, descPats []*regexp.Regexp) *Node { + var hits []Node + for _, n := range nodes { + if n.Desc == "" { + continue + } + for _, p := range descPats { + if p.MatchString(n.Desc) { + hits = append(hits, n) + break + } + } + } + if len(hits) == 0 { + return nil + } + best := hits[0] + for _, h := range hits[1:] { + if h.Area() < best.Area() { + best = h + } + } + return &best +} + +// WaitFor polls DumpUI until a match appears or timeout. +func (c *Client) WaitFor(cacheDir string, textPats, descPats []*regexp.Regexp, timeout time.Duration) (*Node, error) { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + nodes, err := c.DumpUI(cacheDir) + if err == nil { + if n := FindLargest(nodes, textPats, descPats); n != nil { + return n, nil + } + } + time.Sleep(800 * time.Millisecond) + } + return nil, fmt.Errorf("ui node not found within %s", timeout) +} + +// PlaybackActive is true when the package media session is PLAYING. +func (c *Client) PlaybackActive(pkg string) bool { + out := c.Out("shell", "dumpsys", "media_session") + if !strings.Contains(out, pkg) { + return false + } + return strings.Contains(out, "state=PLAYING") || strings.Contains(out, "PLAYING(") +} diff --git a/apps/pkg/app/app.go b/apps/pkg/app/app.go new file mode 100644 index 0000000..431b979 --- /dev/null +++ b/apps/pkg/app/app.go @@ -0,0 +1,154 @@ +package app + +import ( + "flag" + "fmt" + "sort" + "sync" + + "drmdecryption/adb" + "drmdecryption/capture" + "drmdecryption/mpd" + "drmdecryption/session" +) + +// Channel is a named live target inside an app (e.g. rteone). +type Channel struct { + ID string + Label string +} + +// App is a per-streaming-app plugin, compiled in from apps/modules/. The +// capture core stays generic: an app supplies its package name, UI automation, +// capture hints, key mode and optional MPD rewrite. +type App interface { + Name() string + Package() string + LicenseURL() string + Launch(c *adb.Client) error + AutoPlay(c *adb.Client, channel string) error + CaptureHints() capture.Hints + MPDRewriter() mpd.Rewriter + Channels() []Channel + HasChannel(id string) bool + + // ProxyBin is the on-device MITM binary to push. + ProxyBin() string + // CAHash is the MITM CA subject hash to reinject. + CAHash() string + // KeyMode selects the wvkey mode: "modulardrm" or "raw". + KeyMode() string +} + +// Catalog is implemented by apps whose channels can be resolved without a phone +// (a public playback catalog). Drives the `catalog` subcommand. +type Catalog interface { + // Resolve returns streamd-ready credentials for a channel id. + Resolve(channel string) (session.Stream, error) + // EnrichWithKeys fills PSSH + KID:KEY using the local CDM. + EnrichWithKeys(info *session.Stream, python, wvd string) error +} + +// CatalogRow is one channel in a catalog listing. +type CatalogRow struct { + ID string + CatalogID string + Label string +} + +// CatalogLister is the optional richer listing a catalog app may provide: each +// channel's resolved catalog id plus whatever feature flags the provider +// publishes alongside it. +type CatalogLister interface { + ListChannels() (rows []CatalogRow, flags map[string]bool, err error) +} + +// StreamDefaults is implemented by apps that need particular downloader settings. +// It replaces name-sniffing in the media supervisor. +type StreamDefaults interface { + // VideoSelect / AudioSelect are N_m3u8DL-RE selector expressions. + VideoSelect() string + AudioSelect() string + // RewriterName is the registered mpd rewriter to run, or "none". + RewriterName() string + // LiveWaitSeconds is the downloader's live refresh wait. + LiveWaitSeconds() int + // TSReadyBytes is how much muxed output to buffer before probing. + TSReadyBytes() int64 +} + +// TransparentMITM is implemented by apps that must use iptables REDIRECT capture +// (no Wi‑Fi http_proxy) so a phone VPN (e.g. NordVPN UK for BBC) can stay on. +type TransparentMITM interface { + UseTransparentMITM() bool +} + +var ( + mu sync.RWMutex + registry = map[string]func() (App, error){} +) + +// Register adds a constructor for --app . Called from each module's init(). +func Register(name string, ctor func() (App, error)) { + mu.Lock() + defer mu.Unlock() + registry[name] = ctor +} + +// Get constructs a registered app by name. +func Get(name string) (App, error) { + mu.RLock() + ctor, ok := registry[name] + mu.RUnlock() + if !ok { + return nil, fmt.Errorf("unknown app %q — compiled-in apps: %v", name, Names()) + } + return ctor() +} + +// Names lists registered app ids, sorted. +func Names() []string { + mu.RLock() + defer mu.RUnlock() + out := make([]string, 0, len(registry)) + for k := range registry { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// All constructs every registered app, skipping those that fail to build. +func All() []App { + out := []App{} + for _, n := range Names() { + a, err := Get(n) + if err != nil { + continue + } + out = append(out, a) + } + return out +} + +// flagHooks are registered from each module's init() so overrides can be bound +// to the module's config holder BEFORE any app is constructed. +var flagHooks []func(*flag.FlagSet) + +// RegisterFlags records a module's flag binder. Called from init(). +func RegisterFlags(bind func(*flag.FlagSet)) { + mu.Lock() + defer mu.Unlock() + flagHooks = append(flagHooks, bind) +} + +// BindFlags lets every compiled-in module register its value overrides on fs. +// Call this before fs.Parse, and construct apps only afterwards. +func BindFlags(fs *flag.FlagSet) { + mu.RLock() + hooks := append([]func(*flag.FlagSet){}, flagHooks...) + mu.RUnlock() + for _, h := range hooks { + h(fs) + } +} diff --git a/apps/pkg/brightcove/brightcove.go b/apps/pkg/brightcove/brightcove.go new file mode 100644 index 0000000..49ff703 --- /dev/null +++ b/apps/pkg/brightcove/brightcove.go @@ -0,0 +1,161 @@ +// Package brightcove talks to the Brightcove Playback API and reads Widevine +// material out of an HLS master. Brightcove is a platform, not a provider: this +// package holds no account, policy key, video id or channel knowledge. +package brightcove + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" +) + +// EdgeAPI is the Playback API base. Overridable for tests or a proxy. +var EdgeAPI = "https://edge.api.brightcove.com/playback/v1" + +var ( + // SESSION-KEY / KEY lines put URI and KEYFORMAT in either order. + reWidevinePSSH = regexp.MustCompile(`(?is)KEYFORMAT="urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed".*?URI="data:text/plain;base64,([A-Za-z0-9+/=]+)"`) + reWidevinePSSH2 = regexp.MustCompile(`(?is)URI="data:text/plain;base64,([A-Za-z0-9+/=]+)".*?KEYFORMAT="urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed"`) + reKeyID = regexp.MustCompile(`(?i)keyId=([0-9a-f]{32})`) +) + +// WidevineKeySystem is the key_systems map key for Widevine sources. +const WidevineKeySystem = "com.widevine.alpha" + +// Video is the subset of a Playback API response we use. +type Video struct { + ID string `json:"id"` + Name string `json:"name"` + Sources []struct { + Type string `json:"type"` + Src string `json:"src"` + KeySystems map[string]struct { + LicenseURL string `json:"license_url"` + } `json:"key_systems"` + } `json:"sources"` +} + +// PlaybackURL is the canonical (token-free) Playback API URL for a video. +func PlaybackURL(accountID, videoID string) string { + return fmt.Sprintf("%s/accounts/%s/videos/%s", EdgeAPI, accountID, videoID) +} + +// FetchPlayback resolves a video through the Playback API. livePlaybackToken may +// be empty for assets that do not need one. +func FetchPlayback(accountID, videoID, livePlaybackToken, policyKey string) (Video, error) { + var v Video + if policyKey == "" { + return v, fmt.Errorf("policy key required") + } + if accountID == "" { + return v, fmt.Errorf("account id required") + } + u := PlaybackURL(accountID, videoID) + if livePlaybackToken != "" { + u += "?livePlaybackToken=" + livePlaybackToken + } + req, err := http.NewRequest(http.MethodGet, u, nil) + if err != nil { + return v, err + } + req.Header.Set("Accept", "application/json;pk="+policyKey) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return v, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 { + return v, fmt.Errorf("brightcove playback HTTP %d: %s", resp.StatusCode, Trim(string(body), 200)) + } + if err := json.Unmarshal(body, &v); err != nil { + return v, err + } + return v, nil +} + +// PickHLSAndLicense chooses the best HLS source: prefer a DVR playlist that +// carries a Widevine license URL, then any Widevine source, then any DVR +// playlist, then any HLS at all. +func PickHLSAndLicense(v Video) (hls, license string) { + var dvr, dvrWV, anyWV, anyHLS string + var dvrLic, anyLic string + for _, s := range v.Sources { + if !strings.Contains(strings.ToLower(s.Type), "mpegurl") && !strings.Contains(strings.ToLower(s.Src), ".m3u8") { + continue + } + src := s.Src + lic := "" + if ks, ok := s.KeySystems[WidevineKeySystem]; ok { + lic = ks.LicenseURL + } + isDVR := strings.Contains(src, "playlist-hls-dvr.m3u8") || strings.Contains(src, "-dvr") + switch { + case isDVR && lic != "" && dvrWV == "": + dvrWV, dvrLic = src, lic + case isDVR && dvr == "": + dvr = src + case lic != "" && anyWV == "": + anyWV, anyLic = src, lic + case anyHLS == "": + anyHLS = src + } + } + switch { + case dvrWV != "": + return dvrWV, dvrLic + case anyWV != "": + return anyWV, anyLic + case dvr != "": + return dvr, "" + default: + return anyHLS, "" + } +} + +// ExtractWidevinePSSH returns the base64 Widevine init data from an HLS master. +func ExtractWidevinePSSH(master string) string { + if m := reWidevinePSSH.FindStringSubmatch(master); len(m) == 2 { + return m[1] + } + if m := reWidevinePSSH2.FindStringSubmatch(master); len(m) == 2 { + return m[1] + } + return "" +} + +// PrimaryKID returns the first keyId= seen in an HLS master, lowercased. +func PrimaryKID(master string) string { + if kids := reKeyID.FindAllStringSubmatch(master, -1); len(kids) > 0 { + return strings.ToLower(kids[0][1]) + } + return "" +} + +// Get fetches a URL as text (used for the HLS master). +func Get(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return "", fmt.Errorf("HTTP %d", resp.StatusCode) + } + b, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(b), nil +} + +// Trim shortens a string for error messages. +func Trim(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "..." +} diff --git a/apps/pkg/capture/capture.go b/apps/pkg/capture/capture.go new file mode 100644 index 0000000..35ee985 --- /dev/null +++ b/apps/pkg/capture/capture.go @@ -0,0 +1,284 @@ +package capture + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "drmdecryption/adb" +) + +// Hints tell the waiter which fields / log tags mean a capture is complete. +type Hints struct { + RemoteCaps []string // paths on device to pull + RemoteLog string + Require []string // json keys: auth, pid, pssh, mpd + // BestEffort: return early when any useful field appears; at timeout return + // whatever was collected (error only if completely empty). + BestEffort bool + MPDLogRE *regexp.Regexp + // Score ranks candidate manifest URLs. Zero value falls back to + // DefaultScoreCfg, which has no provider hosts. + Score ScoreCfg +} + +// RemoteCapPaths are the on-device locations the MITM may persist its capture +// JSON to. The rte_cap.json entries are legacy names still written by proxies +// already deployed on phones in the field. +var RemoteCapPaths = []string{ + "/data/local/tmp/appproxy_cap.json", + "/data/local/tmp/rte_cap.json", + "/sdcard/Download/rte_cap.json", + "/storage/emulated/0/Download/rte_cap.json", +} + +// RemoteLogPath is where the on-device MITM writes its log. +const RemoteLogPath = "/data/local/tmp/appproxy.log" + +// DefaultHints is the device-layout baseline every app starts from. Callers set +// Require (and optionally Score) for their own DRM shape. +func DefaultHints() Hints { + return Hints{ + RemoteCaps: append([]string{}, RemoteCapPaths...), + RemoteLog: RemoteLogPath, + MPDLogRE: regexp.MustCompile(`\[MPD\] (https://\S+)`), + Score: DefaultScoreCfg(), + } +} + +// DefaultPassiveHints is for a bare capture run: dump whatever the MITM sees. +func DefaultPassiveHints() Hints { + h := DefaultHints() + h.BestEffort = true + return h +} + +// score applies the hints' URL scoring, defaulting when unset. +func (h Hints) score(u string) int { + if len(h.Score.Deny) == 0 && len(h.Score.Hosts) == 0 { + return DefaultScoreCfg().Score(u) + } + return h.Score.Score(u) +} + +// Data is the merged capture payload before key fetch. +type Data map[string]string + +func (d Data) Get(k string) string { return d[k] } + +// Wait pulls device JSON + local mirrored log until required fields exist. +func Wait(c *adb.Client, hints Hints, localLog string, localCap string, timeout time.Duration) (Data, error) { + deadline := time.Now().Add(timeout) + lastNote := "" + for time.Now().Before(deadline) { + cap := Data{} + for _, remote := range hints.RemoteCaps { + _ = os.Remove(localCap) + if err := c.Pull(remote, localCap); err != nil { + continue + } + raw, err := os.ReadFile(localCap) + if err != nil { + continue + } + var m map[string]any + if json.Unmarshal(raw, &m) != nil { + continue + } + for k, v := range m { + if s, ok := v.(string); ok && s != "" { + cap[k] = s + } + } + break + } + + logText := "" + if b, err := os.ReadFile(localLog); err == nil { + logText = string(b) + } + enrichFromProxyLog(cap, logText, hints) + + if hints.BestEffort { + if hasUseful(cap, hints) { + _ = os.WriteFile(localCap, mustJSON(cap), 0o644) + return cap, nil + } + } else { + missing := false + for _, k := range hints.Require { + if strings.TrimSpace(cap[k]) == "" { + missing = true + break + } + } + if !missing { + _ = os.WriteFile(localCap, mustJSON(cap), 0o644) + return cap, nil + } + } + + have := []string{} + keys := hints.Require + if hints.BestEffort { + keys = []string{"mpd", "license_url", "pssh", "auth", "pid"} + } + for _, k := range keys { + if cap[k] != "" { + have = append(have, k) + } + } + note := "json=" + strings.Join(have, ",") + if note == "json=" { + note = "json=none" + } + if strings.Contains(logText, "[LIC]") { + note += " log=LIC" + } + if strings.Contains(logText, "[PSSH]") { + note += " log=PSSH" + } + if strings.Contains(logText, "[MPD]") || strings.Contains(logText, "[MAN]") || strings.Contains(logText, "[MEDIA]") || strings.Contains(logText, "[MS]") { + note += " log=MAN" + } + if note != lastNote { + fmt.Println(" …" + note) + lastNote = note + } + time.Sleep(time.Second) + } + if hints.BestEffort { + // Final pull after timeout — return whatever we got. + cap := Data{} + for _, remote := range hints.RemoteCaps { + _ = os.Remove(localCap) + if err := c.Pull(remote, localCap); err != nil { + continue + } + raw, err := os.ReadFile(localCap) + if err != nil { + continue + } + var m map[string]any + if json.Unmarshal(raw, &m) != nil { + continue + } + for k, v := range m { + if s, ok := v.(string); ok && s != "" { + cap[k] = s + } + } + break + } + logText := "" + if b, err := os.ReadFile(localLog); err == nil { + logText = string(b) + } + enrichFromProxyLog(cap, logText, hints) + if len(cap) > 0 { + _ = os.WriteFile(localCap, mustJSON(cap), 0o644) + return cap, nil + } + return nil, fmt.Errorf("timed out — no DRM/manifest traffic seen (play something on the phone)") + } + return nil, fmt.Errorf("timed out waiting for capture (%v)", hints.Require) +} + +func hasUseful(cap Data, hints Hints) bool { + // Real stream signal only — ignore catalog/EPG URLs parked in "mpd". + if mpd := strings.TrimSpace(cap["mpd"]); mpd != "" && hints.score(mpd) > 0 { + return true + } + for _, k := range []string{"license_url", "pssh", "auth", "pid"} { + if strings.TrimSpace(cap[k]) != "" { + return true + } + } + return false +} + +var ( + reLogLIC = regexp.MustCompile(`\[LIC\]\s+(?:POST|GET)\s+(https://\S+)`) + reLogMPD = regexp.MustCompile(`\[MPD\]\s+(https://\S+)`) + reLogMAN = regexp.MustCompile(`\[MAN\]\s+GET\s+(https://\S+)`) + reLogMEDIA = regexp.MustCompile(`\[MEDIA\]\s+(https://\S+)`) + reLogPSSH = regexp.MustCompile(`\[PSSH\]\s+(\S+)`) +) + +func enrichFromProxyLog(cap Data, logText string, hints Hints) { + if logText == "" { + return + } + bestMPD := "" + bestScore := 0 + consider := func(u string) { + u = strings.TrimRight(u, ".,)") + if u == "" || !strings.HasPrefix(u, "http") { + return + } + sc := hints.score(u) + // Ignore EPG/schedule/metrics noise (score <= 0). + if sc <= 0 { + return + } + if sc > bestScore { + bestScore = sc + bestMPD = u + } + } + if hints.MPDLogRE != nil { + for _, m := range hints.MPDLogRE.FindAllStringSubmatch(logText, -1) { + consider(m[1]) + } + } + for _, m := range reLogMPD.FindAllStringSubmatch(logText, -1) { + consider(m[1]) + } + for _, m := range reLogMAN.FindAllStringSubmatch(logText, -1) { + consider(m[1]) + } + for _, m := range reLogMEDIA.FindAllStringSubmatch(logText, -1) { + consider(m[1]) + } + if bestMPD != "" && (cap["mpd"] == "" || hints.score(bestMPD) > hints.score(cap["mpd"])) { + cap["mpd"] = bestMPD + } + // Drop a previously stored non-manifest "mpd" (e.g. schedules feed). + if cap["mpd"] != "" && hints.score(cap["mpd"]) <= 0 { + delete(cap, "mpd") + } + if cap["license_url"] == "" { + if ms := reLogLIC.FindAllStringSubmatch(logText, -1); len(ms) > 0 { + cap["license_url"] = strings.TrimRight(ms[len(ms)-1][1], ".,)") + } + } + if cap["pssh"] == "" { + if ms := reLogPSSH.FindAllStringSubmatch(logText, -1); len(ms) > 0 { + cap["pssh"] = ms[len(ms)-1][1] + } + } +} + +func mustJSON(d Data) []byte { + b, _ := json.MarshalIndent(map[string]string(d), "", " ") + return append(b, '\n') +} + +// MirrorLog starts a background `adb exec-out log` equivalent via continuous pull. +// For v1 we periodically pull the remote log file into localLog. +func MirrorLogLoop(c *adb.Client, remote, local string, stop <-chan struct{}) { + _ = os.MkdirAll(filepath.Dir(local), 0o755) + for { + select { + case <-stop: + return + default: + _ = c.Pull(remote, local) + time.Sleep(800 * time.Millisecond) + } + } +} diff --git a/apps/pkg/capture/score.go b/apps/pkg/capture/score.go new file mode 100644 index 0000000..ce1d2df --- /dev/null +++ b/apps/pkg/capture/score.go @@ -0,0 +1,100 @@ +package capture + +import "strings" + +// ScoreCfg ranks candidate URLs seen by the MITM so a schedule/EPG/analytics URL +// is never mistaken for a stream manifest. Providers supply their own hosts and +// noise needles; this package ships only format-level scoring. +type ScoreCfg struct { + // Deny: any match scores the URL out entirely. + Deny []string `yaml:"deny"` + // Hosts: known-good manifest hosts for this provider. + Hosts []string `yaml:"hosts"` + // HostBonus is added when a Hosts entry matches (default 50). + HostBonus int `yaml:"host_bonus"` +} + +// DefaultScoreCfg is the provider-neutral baseline: catalog/analytics shapes that +// are never a manifest for anyone. +func DefaultScoreCfg() ScoreCfg { + return ScoreCfg{ + Deny: []string{ + "schedules", "bylistingtime", "maxlistings", "bycallsign", + "/feed.", "playback_config", "config.json", + }, + HostBonus: 50, + } +} + +// Merge overlays a provider's hosts and extra deny needles on the baseline. +func (s ScoreCfg) Merge(other ScoreCfg) ScoreCfg { + out := s + out.Deny = append(append([]string{}, s.Deny...), other.Deny...) + out.Hosts = append(append([]string{}, s.Hosts...), other.Hosts...) + if other.HostBonus != 0 { + out.HostBonus = other.HostBonus + } + if out.HostBonus == 0 { + out.HostBonus = 50 + } + return out +} + +// Score rates a URL as a live manifest. Zero or negative means "not a manifest". +func (s ScoreCfg) Score(u string) int { + lu := strings.ToLower(u) + path := lu + if i := strings.Index(path, "?"); i >= 0 { + path = path[:i] + } + for _, bad := range s.Deny { + if bad != "" && strings.Contains(lu, strings.ToLower(bad)) { + return -100 + } + } + score := 0 + bonus := s.HostBonus + if bonus == 0 { + bonus = 50 + } + for _, host := range s.Hosts { + if host != "" && strings.Contains(lu, strings.ToLower(host)) { + score += bonus + break + } + } + // Format-level signals — true for any provider. + if strings.Contains(path, "playlist-hls") || strings.Contains(path, "playlist.m3u8") { + score += 40 + } + if strings.Contains(path, "chunklist") { + score += 10 + } + if strings.HasSuffix(path, ".m3u8") { + score += 5 + } + if strings.HasSuffix(path, ".mpd") || strings.Contains(path, "manifest.mpd") { + score += 30 + } + if strings.Contains(path, ".isml") || strings.Contains(path, "/manifest") { + score += 20 + } + return score +} + +// ProxyArgs renders this config as flags for the on-device MITM, which runs on +// the phone and cannot read an app module's values file itself. +func (s ScoreCfg) ProxyArgs() []string { + out := []string{} + for _, h := range s.Hosts { + if h != "" { + out = append(out, "-score-host", h) + } + } + for _, d := range s.Deny { + if d != "" { + out = append(out, "-score-deny", d) + } + } + return out +} diff --git a/apps/pkg/capture/score_test.go b/apps/pkg/capture/score_test.go new file mode 100644 index 0000000..ae9cf20 --- /dev/null +++ b/apps/pkg/capture/score_test.go @@ -0,0 +1,72 @@ +package capture + +import "testing" + +// provider-shaped config, supplied the way a module would. +func testScoreCfg() ScoreCfg { + return DefaultScoreCfg().Merge(ScoreCfg{ + Deny: []string{"feed.entertainment", "metrics.example", "noise.example"}, + Hosts: []string{"live.example.com", "fastly.live.example.com"}, + }) +} + +func TestScoreCfgScore(t *testing.T) { + cfg := testScoreCfg() + cases := []struct { + url string + want string // "pos" or "neg" + }{ + {"https://feed.entertainment.tv.example.eu/f/1uC-gC/prd-all-schedules?byListingTime=1~2", "neg"}, + {"https://metrics.example.com/v2/tracker?foo=.m3u8", "neg"}, + {"https://cdn.example.com/smarttv/playback_config.json", "neg"}, + {"https://live.example.com/live/foo/manifest.mpd", "pos"}, + {"https://fastly.live.example.com/x/playlist-hls.m3u8", "pos"}, + {"https://unknown-host.test/video/master.m3u8", "pos"}, + {"https://unknown-host.test/live/vc11.isml/Manifest", "pos"}, + } + for _, tc := range cases { + sc := cfg.Score(tc.url) + if tc.want == "pos" && sc <= 0 { + t.Errorf("score(%q)=%d, want > 0", tc.url, sc) + } + if tc.want == "neg" && sc > 0 { + t.Errorf("score(%q)=%d, want <= 0", tc.url, sc) + } + } +} + +// A known provider host must outrank a bare manifest on an unknown host, so the +// best candidate in a noisy MITM log is the provider's own origin. +func TestScoreCfgPrefersKnownHost(t *testing.T) { + cfg := testScoreCfg() + known := cfg.Score("https://live.example.com/live/foo/manifest.mpd") + unknown := cfg.Score("https://somewhere.test/live/foo/manifest.mpd") + if known <= unknown { + t.Fatalf("known host %d should outrank unknown host %d", known, unknown) + } +} + +func TestDefaultScoreCfgNeedsNoProviderHosts(t *testing.T) { + cfg := DefaultScoreCfg() + if cfg.Score("https://anything.test/playlist.m3u8") <= 0 { + t.Fatal("format-level scoring must work without provider hosts") + } + if cfg.Score("https://anything.test/api/schedules?x=1") > 0 { + t.Fatal("schedule feeds must score out with the baseline config") + } +} + +func TestHasUsefulIgnoresSchedulesMPD(t *testing.T) { + hints := DefaultPassiveHints() + hints.Score = testScoreCfg() + cap := Data{ + "mpd": "https://feed.entertainment.tv.example.eu/f/1uC-gC/prd-all-schedules?byListingTime=1~2", + } + if hasUseful(cap, hints) { + t.Fatal("schedules feed must not count as useful capture") + } + cap["license_url"] = "https://widevine.example/license" + if !hasUseful(cap, hints) { + t.Fatal("license_url should count as useful") + } +} diff --git a/apps/pkg/go.mod b/apps/pkg/go.mod new file mode 100644 index 0000000..c23f9f3 --- /dev/null +++ b/apps/pkg/go.mod @@ -0,0 +1,5 @@ +module drmdecryption + +go 1.25.0 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/apps/pkg/go.sum b/apps/pkg/go.sum new file mode 100644 index 0000000..a62c313 --- /dev/null +++ b/apps/pkg/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/apps/pkg/modcfg/modcfg.go b/apps/pkg/modcfg/modcfg.go new file mode 100644 index 0000000..e91b62e --- /dev/null +++ b/apps/pkg/modcfg/modcfg.go @@ -0,0 +1,90 @@ +// Package modcfg loads an app module's values from its local, untracked +// apps/modules//module.yaml, with environment-variable overrides. +// +// Compiled-in module code is tracked in git and must contain no account ids, +// policy keys, video ids, license URLs or origin hostnames. Those live here, in +// a file git ignores, and can also be supplied by flag or environment so a run +// needs no file at all. +package modcfg + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "drmdecryption/repo" + "gopkg.in/yaml.v3" +) + +// Dir returns the directory holding a module's local values. +func Dir(name string) string { + return filepath.Join(repo.Root(), "apps", "modules", name) +} + +// Path returns the module's values file path. +func Path(name string) string { + return filepath.Join(Dir(name), "module.yaml") +} + +// Result reports where a module's values came from, for `drm modules`. +type Result struct { + Path string + Loaded bool +} + +// Load reads apps/modules//module.yaml into dst. A missing file is not an +// error — the module runs on its Go defaults plus flags/env. +func Load(name string, dst any) (Result, error) { + path := Path(name) + res := Result{Path: path} + b, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return res, nil + } + return res, err + } + if err := yaml.Unmarshal(b, dst); err != nil { + return res, fmt.Errorf("%s: %w", path, err) + } + res.Loaded = true + return res, nil +} + +// EnvKey is the environment variable a field override is read from: +// TG4_POLICY_KEY for module "tg4", field "policy_key". +func EnvKey(module, field string) string { + clean := func(s string) string { + s = strings.ToUpper(s) + s = strings.ReplaceAll(s, "-", "_") + s = strings.ReplaceAll(s, ".", "_") + return s + } + return clean(module) + "_" + clean(field) +} + +// Env returns the environment override for a field, or "". +func Env(module, field string) string { + return strings.TrimSpace(os.Getenv(EnvKey(module, field))) +} + +// Override picks the first non-empty of: flag value, environment, current value. +// Use it in a module's config resolution so precedence is uniform. +func Override(module, field, flagVal, current string) string { + if strings.TrimSpace(flagVal) != "" { + return flagVal + } + if v := Env(module, field); v != "" { + return v + } + return current +} + +// Resolve makes a module-relative path absolute against the repo root. +func Resolve(p string) string { + if p == "" || filepath.IsAbs(p) { + return p + } + return filepath.Join(repo.Root(), p) +} diff --git a/apps/pkg/mpd/kid.go b/apps/pkg/mpd/kid.go new file mode 100644 index 0000000..2d2abd3 --- /dev/null +++ b/apps/pkg/mpd/kid.go @@ -0,0 +1,50 @@ +package mpd + +import ( + "encoding/xml" + "fmt" + "regexp" + "strings" +) + +var reDefaultKID = regexp.MustCompile(`(?i)default_KID="([^"]+)"`) + +// KIDToUUID normalizes any hex-ish KID spelling to dashed UUID form. +func KIDToUUID(kid string) (string, error) { + var b strings.Builder + for _, r := range kid { + if (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') { + b.WriteRune(r) + } + } + h := strings.ToLower(b.String()) + if len(h) != 32 { + return "", fmt.Errorf("invalid KID %q", kid) + } + return fmt.Sprintf("%s-%s-%s-%s-%s", h[0:8], h[8:12], h[12:16], h[16:20], h[20:32]), nil +} + +// KIDHex strips dashes and lowercases a KID (the form used as a map key). +func KIDHex(kid string) string { + return strings.ToLower(strings.ReplaceAll(kid, "-", "")) +} + +// ExtractDefaultKID pulls cenc:default_KID out of an MPD, as dashed UUID. +func ExtractDefaultKID(xmlText string) string { + m := reDefaultKID.FindStringSubmatch(xmlText) + if len(m) < 2 { + return "" + } + u, err := KIDToUUID(m[1]) + if err != nil { + return "" + } + return u +} + +// XMLEscape escapes text for inclusion in an XML element body. +func XMLEscape(s string) string { + var b strings.Builder + _ = xml.EscapeText(&b, []byte(s)) + return b.String() +} diff --git a/apps/pkg/mpd/mpd.go b/apps/pkg/mpd/mpd.go new file mode 100644 index 0000000..aed15a1 --- /dev/null +++ b/apps/pkg/mpd/mpd.go @@ -0,0 +1,15 @@ +package mpd + +// Rewriter transforms an upstream live MPD into something N_m3u8DL-RE can +// refresh safely. Apps that need DAI/colon stripping or synthetic timelines +// implement this; others return Passthrough. +type Rewriter interface { + Name() string + Rewrite(upstreamXML []byte, kidHint string) (out []byte, err error) +} + +// Passthrough leaves the MPD unchanged. +type Passthrough struct{} + +func (Passthrough) Name() string { return "none" } +func (Passthrough) Rewrite(in []byte, _ string) ([]byte, error) { return in, nil } diff --git a/apps/pkg/mpd/registry.go b/apps/pkg/mpd/registry.go new file mode 100644 index 0000000..9abefcc --- /dev/null +++ b/apps/pkg/mpd/registry.go @@ -0,0 +1,59 @@ +package mpd + +import ( + "sort" + "strings" + "sync" +) + +var ( + regMu sync.RWMutex + registry = map[string]func() Rewriter{} +) + +func init() { + Register("none", func() Rewriter { return Passthrough{} }) +} + +// Register makes a rewriter resolvable by name, so consumers (streamd) can pick +// one from stored config without importing the provider that implements it. A +// factory is registered rather than an instance because a rewriter may carry +// per-stream fallback state. +func Register(name string, newRewriter func() Rewriter) { + regMu.Lock() + defer regMu.Unlock() + registry[strings.ToLower(name)] = newRewriter +} + +// Lookup builds a fresh rewriter for a name. An empty name, "none" or an unknown +// name yields Passthrough with ok=false, so callers can tell "no rewrite needed" +// from "rewrite with X". +func Lookup(name string) (Rewriter, bool) { + key := strings.ToLower(strings.TrimSpace(name)) + if key == "" || key == "none" { + return Passthrough{}, false + } + regMu.RLock() + newRewriter, ok := registry[key] + regMu.RUnlock() + if !ok { + return Passthrough{}, false + } + rw := newRewriter() + if _, isPass := rw.(Passthrough); isPass { + return rw, false + } + return rw, true +} + +// Names lists registered rewriter names. +func Names() []string { + regMu.RLock() + defer regMu.RUnlock() + out := make([]string, 0, len(registry)) + for k := range registry { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/apps/pkg/mpd/server.go b/apps/pkg/mpd/server.go new file mode 100644 index 0000000..250b067 --- /dev/null +++ b/apps/pkg/mpd/server.go @@ -0,0 +1,104 @@ +package mpd + +import ( + "fmt" + "io" + "net" + "net/http" + "strings" + "time" +) + +// KIDResolver is implemented by rewriters that can recover the live channel from +// a KID alone. The local server uses it so a rewrite still works after the +// upstream (ad-stitched) manifest session has expired and returns 410. +type KIDResolver interface { + ResolveKID(kidHex string) (channel, originBase string, ok bool) +} + +// Primer is implemented by rewriters that want to be told the fallback channel +// and origin discovered on earlier requests. +type Primer interface { + Prime(channel, originBase string) +} + +// LocalServer serves a rewritten live MPD on 127.0.0.1 for a downloader to +// refresh. Every GET re-fetches upstream and re-runs the rewriter. +type LocalServer struct { + URL string + Upstream string + Key string // KID:KEY + Headers map[string]string + + rw Rewriter + kidHint string + ln net.Listener + httpServer *http.Server +} + +// StartLocal starts the rewrite proxy. rw must not be nil; pass Passthrough{} +// to serve upstream unchanged. +func StartLocal(upstream, key string, headers map[string]string, rw Rewriter) (*LocalServer, error) { + if rw == nil { + rw = Passthrough{} + } + s := &LocalServer{Upstream: upstream, Key: key, Headers: headers, rw: rw} + if key != "" && strings.Contains(key, ":") { + s.kidHint = strings.SplitN(key, ":", 2)[0] + if r, ok := rw.(KIDResolver); ok { + if ch, base, found := r.ResolveKID(KIDHex(s.kidHint)); found { + if p, ok := rw.(Primer); ok { + p.Prime(ch, base) + } + } + } + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + s.ln = ln + mux := http.NewServeMux() + mux.HandleFunc("/", s.handle) + s.httpServer = &http.Server{Handler: mux} + s.URL = fmt.Sprintf("http://%s/manifest.mpd", ln.Addr().String()) + go func() { _ = s.httpServer.Serve(ln) }() + return s, nil +} + +func (s *LocalServer) Close() { + if s.httpServer != nil { + _ = s.httpServer.Close() + } + if s.ln != nil { + _ = s.ln.Close() + } +} + +func (s *LocalServer) handle(w http.ResponseWriter, r *http.Request) { + body := "" + req, err := http.NewRequest(http.MethodGet, s.Upstream, nil) + if err == nil { + for k, v := range s.Headers { + req.Header.Set(k, v) + } + client := &http.Client{Timeout: 20 * time.Second} + if resp, err := client.Do(req); err == nil { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + _ = resp.Body.Close() + if resp.StatusCode >= 200 && resp.StatusCode < 300 && len(b) > 0 { + body = string(b) + } + } + } + // Upstream being gone is not fatal: a rewriter with a KID fallback can still + // publish the encoder timeline so the downloader keeps pulling segments. + payload, err := s.rw.Rewrite([]byte(body), s.kidHint) + if err != nil { + http.Error(w, err.Error(), 502) + return + } + w.Header().Set("Content-Type", "application/dash+xml") + w.WriteHeader(200) + _, _ = w.Write(payload) +} diff --git a/apps/pkg/mpd/smooth.go b/apps/pkg/mpd/smooth.go new file mode 100644 index 0000000..76a3531 --- /dev/null +++ b/apps/pkg/mpd/smooth.go @@ -0,0 +1,142 @@ +package mpd + +import ( + "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" + "sync" +) + +// DefaultMSSTimescale is the Smooth Streaming tick rate (100ns units). +const DefaultMSSTimescale = 10_000_000 + +var ( + reSegDur = regexp.MustCompile(`\bd="(\d+)"`) + reSegT = regexp.MustCompile(`\bt="(\d+)"`) + reSegR = regexp.MustCompile(`\br="(\d+)"`) + reSegEntry = regexp.MustCompile(``) +) + +// ScaleMSS converts a Smooth Streaming tick to another timescale without +// overflowing int64 (MSS times are ~1e16; mss*48000 does not fit). +func ScaleMSS(mss, fromScale, toScale int64) int64 { + if fromScale <= 0 { + fromScale = DefaultMSSTimescale + } + return mss/fromScale*toScale + (mss%fromScale)*toScale/fromScale +} + +// Grid is the segment duration + phase of one encoder channel, per media type. +type Grid struct { + VDur, ADur int64 + VMod, AMod int64 +} + +// GridSpec describes how to learn a Grid from an origin's Smooth manifest. +type GridSpec struct { + // ManifestPath is appended to the origin base (e.g. "Manifest"). + ManifestPath string + // VideoName / AudioName are StreamIndex Name (or Type) attributes. + VideoName, AudioName string + // Timescales the DASH output uses. + VideoTimescale, AudioTimescale int64 + MSSTimescale int64 + // FallbackSegmentDuration in MSS ticks when the manifest has no d="". + FallbackSegmentDuration int64 +} + +var ( + gridMu sync.Mutex + gridCache = map[string]Grid{} +) + +// LearnGrid fetches the origin Smooth manifest once per channel and derives the +// segment duration and phase offset for video and audio. +func LearnGrid(channel, originBase string, spec GridSpec) (Grid, error) { + gridMu.Lock() + if g, ok := gridCache[channel]; ok { + gridMu.Unlock() + return g, nil + } + gridMu.Unlock() + + if !strings.HasSuffix(originBase, "/") { + originBase += "/" + } + url := originBase + spec.ManifestPath + resp, err := http.Get(url) + if err != nil { + return Grid{}, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return Grid{}, err + } + text := string(body) + video := ParseMSSStreamTimes(text, spec.VideoName) + audio := ParseMSSStreamTimes(text, spec.AudioName) + mssDur := spec.FallbackSegmentDuration + if m := reSegDur.FindStringSubmatch(text); len(m) == 2 { + if v, e := strconv.ParseInt(m[1], 10, 64); e == nil { + mssDur = v + } + } + if len(video) == 0 || len(audio) == 0 { + return Grid{}, fmt.Errorf("encoder Manifest missing streams at %s", url) + } + ms := spec.MSSTimescale + vDur := ScaleMSS(mssDur, ms, spec.VideoTimescale) + aDur := ScaleMSS(mssDur, ms, spec.AudioTimescale) + g := Grid{ + VDur: vDur, + ADur: aDur, + VMod: ScaleMSS(video[len(video)-1], ms, spec.VideoTimescale) % vDur, + AMod: ScaleMSS(audio[len(audio)-1], ms, spec.AudioTimescale) % aDur, + } + gridMu.Lock() + gridCache[channel] = g + gridMu.Unlock() + return g, nil +} + +// ParseMSSStreamTimes expands the timeline of one StreamIndex into +// absolute MSS tick times. streamName matches Name="" or Type="". +func ParseMSSStreamTimes(manifestXML, streamName string) []int64 { + reBlock := regexp.MustCompile(`(?is)]*\bName="` + regexp.QuoteMeta(streamName) + `"[^>]*>(.*?)`) + m := reBlock.FindStringSubmatch(manifestXML) + if m == nil { + reBlock = regexp.MustCompile(`(?is)]*\bType="` + regexp.QuoteMeta(streamName) + `"[^>]*>(.*?)`) + m = reBlock.FindStringSubmatch(manifestXML) + } + if m == nil { + return nil + } + var times []int64 + tCur := int64(-1) + for _, c := range reSegEntry.FindAllStringSubmatch(m[1], -1) { + attrs := c[1] + tm := reSegT.FindStringSubmatch(attrs) + dm := reSegDur.FindStringSubmatch(attrs) + rm := reSegR.FindStringSubmatch(attrs) + if tm != nil { + tCur, _ = strconv.ParseInt(tm[1], 10, 64) + } + if tCur < 0 || dm == nil { + continue + } + d, _ := strconv.ParseInt(dm[1], 10, 64) + r := int64(0) + if rm != nil { + r, _ = strconv.ParseInt(rm[1], 10, 64) + } + for i := int64(0); i <= r; i++ { + times = append(times, tCur) + tCur += d + } + } + return times +} diff --git a/apps/pkg/mpd/synthetic.go b/apps/pkg/mpd/synthetic.go new file mode 100644 index 0000000..a1aeffd --- /dev/null +++ b/apps/pkg/mpd/synthetic.go @@ -0,0 +1,280 @@ +package mpd + +import ( + "fmt" + "strings" + "sync" + "time" +) + +// SyntheticConfig describes a live encoder's DASH grid well enough to publish a +// synthetic manifest for it. Every value a provider could differ on lives here; +// nothing in this package names a provider. +type SyntheticConfig struct { + // Timescales. + MSSTimescale int64 `yaml:"mss_timescale"` + VideoTimescale int64 `yaml:"video_timescale"` + AudioTimescale int64 `yaml:"audio_timescale"` + + // Smooth Streaming stream names used to learn the segment grid. + ManifestPath string `yaml:"manifest_path"` + VideoName string `yaml:"video_name"` + AudioName string `yaml:"audio_name"` + + // Segment path appended to the origin base for DASH segments. + SegmentPathSuffix string `yaml:"segment_path_suffix"` + + // Representation attributes. + VideoBandwidth int `yaml:"video_bandwidth"` + AudioBandwidth int `yaml:"audio_bandwidth"` + VideoCodecs string `yaml:"video_codecs"` + AudioCodecs string `yaml:"audio_codecs"` + Width int `yaml:"width"` + Height int `yaml:"height"` + AudioSamplingRate int `yaml:"audio_sampling_rate"` + AudioChannels int `yaml:"audio_channels"` + Lang string `yaml:"lang"` + + // Segment naming. "{channel}" is substituted; empty means derive as + // -=[-$Time$].. + VideoInitTemplate string `yaml:"video_init_template"` + VideoMediaTemplate string `yaml:"video_media_template"` + AudioInitTemplate string `yaml:"audio_init_template"` + AudioMediaTemplate string `yaml:"audio_media_template"` + SegmentExt string `yaml:"segment_ext"` + + // Timeline shape. + LiveEdgeOffset float64 `yaml:"live_edge_offset_s"` + WindowSegments int64 `yaml:"window_segments"` + FallbackSegmentDuration int64 `yaml:"fallback_segment_duration"` + + // MPD-level durations, as ISO-8601 strings. + MinUpdatePeriod string `yaml:"min_update_period"` + MinBufferTime string `yaml:"min_buffer_time"` + TimeShiftBufferDepth string `yaml:"time_shift_buffer_depth"` + MaxSegmentDuration string `yaml:"max_segment_duration"` +} + +// WithDefaults fills anything left zero with the common Smooth-to-DASH values. +// These are container mechanics, not provider identity, so defaulting is safe. +func (c SyntheticConfig) WithDefaults() SyntheticConfig { + if c.MSSTimescale == 0 { + c.MSSTimescale = DefaultMSSTimescale + } + if c.VideoTimescale == 0 { + c.VideoTimescale = 600 + } + if c.AudioTimescale == 0 { + c.AudioTimescale = 48_000 + } + if c.ManifestPath == "" { + c.ManifestPath = "Manifest" + } + if c.VideoName == "" { + c.VideoName = "video" + } + if c.AudioName == "" { + c.AudioName = "audio_128k" + } + if c.SegmentPathSuffix == "" { + c.SegmentPathSuffix = "dash/" + } + if c.VideoBandwidth == 0 { + c.VideoBandwidth = 6_000_000 + } + if c.AudioBandwidth == 0 { + c.AudioBandwidth = 128_000 + } + if c.VideoCodecs == "" { + c.VideoCodecs = "avc1.640028" + } + if c.AudioCodecs == "" { + c.AudioCodecs = "mp4a.40.2" + } + if c.Width == 0 { + c.Width = 1920 + } + if c.Height == 0 { + c.Height = 1080 + } + if c.AudioSamplingRate == 0 { + c.AudioSamplingRate = 48_000 + } + if c.AudioChannels == 0 { + c.AudioChannels = 2 + } + if c.Lang == "" { + c.Lang = "en" + } + if c.SegmentExt == "" { + c.SegmentExt = "dash" + } + if c.LiveEdgeOffset == 0 { + c.LiveEdgeOffset = 12 + } + if c.WindowSegments == 0 { + c.WindowSegments = 6 + } + if c.FallbackSegmentDuration == 0 { + c.FallbackSegmentDuration = 38_400_000 + } + if c.MinUpdatePeriod == "" { + c.MinUpdatePeriod = "PT2S" + } + if c.MinBufferTime == "" { + c.MinBufferTime = "PT8S" + } + if c.TimeShiftBufferDepth == "" { + c.TimeShiftBufferDepth = "PT1M" + } + if c.MaxSegmentDuration == "" { + c.MaxSegmentDuration = "PT4S" + } + return c +} + +// GridSpec is the LearnGrid view of this config, exported so a provider module can +// learn the same encoder grid the builder uses. +func (c SyntheticConfig) GridSpec() GridSpec { + return GridSpec{ + ManifestPath: c.ManifestPath, + VideoName: c.VideoName, + AudioName: c.AudioName, + VideoTimescale: c.VideoTimescale, + AudioTimescale: c.AudioTimescale, + MSSTimescale: c.MSSTimescale, + FallbackSegmentDuration: c.FallbackSegmentDuration, + } +} + +func (c SyntheticConfig) segName(tmpl, channel, stream string, bandwidth int, withTime bool) string { + if tmpl != "" { + return strings.ReplaceAll(tmpl, "{channel}", channel) + } + if withTime { + return fmt.Sprintf("%s-%s=%d-$Time$.%s", channel, stream, bandwidth, c.SegmentExt) + } + return fmt.Sprintf("%s-%s=%d.%s", channel, stream, bandwidth, c.SegmentExt) +} + +type anchor struct { + V, A int64 +} + +var ( + anchorMu sync.Mutex + anchors = map[string]*anchor{} +) + +const syntheticTemplate = ` + + %s + + + + + + + + + + + + + + + + + + + + +` + +// BuildSynthetic publishes a dynamic MPD on the encoder's own segment grid, so a +// downloader keeps fetching live segments even once the upstream (ad-stitched) +// manifest session has expired. +func BuildSynthetic(cfg SyntheticConfig, channel, originBase, kid string) ([]byte, error) { + cfg = cfg.WithDefaults() + if !strings.HasSuffix(originBase, "/") { + originBase += "/" + } + segBase := originBase + cfg.SegmentPathSuffix + kidU, err := KIDToUUID(kid) + if err != nil { + return nil, err + } + grid, err := LearnGrid(channel, originBase, cfg.GridSpec()) + if err != nil { + return nil, err + } + // Float seconds so sub-second phase stays close to the encoder. + now := float64(time.Now().UTC().UnixNano())/1e9 - cfg.LiveEdgeOffset + align := func(ts, dur, mod int64) int64 { + raw := int64(now * float64(ts)) + return raw - (raw-mod)%dur + } + vt := align(cfg.VideoTimescale, grid.VDur, grid.VMod) + at := align(cfg.AudioTimescale, grid.ADur, grid.AMod) + maxCount := cfg.WindowSegments + + anchorMu.Lock() + a := anchors[channel] + if a == nil { + a = &anchor{ + V: vt - (maxCount-1)*grid.VDur, + A: at - (maxCount-1)*grid.ADur, + } + anchors[channel] = a + } + vStart, aStart := a.V, a.A + if vt > vStart+(maxCount-1)*grid.VDur { + vStart = vt - (maxCount-1)*grid.VDur + aStart = at - (maxCount-1)*grid.ADur + a.V, a.A = vStart, aStart + } else if vt < vStart { + vStart = vt - (maxCount-1)*grid.VDur + aStart = at - (maxCount-1)*grid.ADur + a.V, a.A = vStart, aStart + } + anchorMu.Unlock() + + vCount := (vt-vStart)/grid.VDur + 1 + aCount := (at-aStart)/grid.ADur + 1 + if vCount < 1 { + vCount = 1 + } + if aCount < 1 { + aCount = 1 + } + if vCount > maxCount { + vStart = vt - (maxCount-1)*grid.VDur + aStart = at - (maxCount-1)*grid.ADur + vCount, aCount = maxCount, maxCount + anchorMu.Lock() + anchors[channel] = &anchor{V: vStart, A: aStart} + anchorMu.Unlock() + } + + published := time.Now().UTC().Format("2006-01-02T15:04:05.000000Z") + vInit := cfg.segName(cfg.VideoInitTemplate, channel, cfg.VideoName, cfg.VideoBandwidth, false) + vMedia := cfg.segName(cfg.VideoMediaTemplate, channel, cfg.VideoName, cfg.VideoBandwidth, true) + aInit := cfg.segName(cfg.AudioInitTemplate, channel, cfg.AudioName, cfg.AudioBandwidth, false) + aMedia := cfg.segName(cfg.AudioMediaTemplate, channel, cfg.AudioName, cfg.AudioBandwidth, true) + + out := fmt.Sprintf(syntheticTemplate, + published, cfg.MinUpdatePeriod, cfg.MinBufferTime, cfg.TimeShiftBufferDepth, cfg.MaxSegmentDuration, + XMLEscape(segBase), + cfg.Lang, kidU, + cfg.VideoBandwidth, cfg.VideoCodecs, cfg.Width, cfg.Height, + cfg.VideoTimescale, vInit, vMedia, vStart, grid.VDur, vCount-1, + cfg.Lang, kidU, + cfg.AudioBandwidth, cfg.AudioCodecs, cfg.AudioSamplingRate, cfg.AudioChannels, + cfg.AudioTimescale, aInit, aMedia, aStart, grid.ADur, aCount-1, + ) + return []byte(out), nil +} diff --git a/apps/pkg/mpd/synthetic_test.go b/apps/pkg/mpd/synthetic_test.go new file mode 100644 index 0000000..6d1f7ea --- /dev/null +++ b/apps/pkg/mpd/synthetic_test.go @@ -0,0 +1,264 @@ +package mpd + +import ( + "fmt" + "net/http" + "net/http/httptest" + "regexp" + "strconv" + "strings" + "testing" +) + +// A Smooth Streaming manifest shaped like a live encoder serves: 4s segments at +// the 10MHz MSS tick rate. +const smoothManifest = ` + + + + + + + + + +` + +func originServer(t *testing.T) string { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, "/Manifest") { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(smoothManifest)) + })) + t.Cleanup(srv.Close) + return srv.URL + "/live/test.isml/" +} + +// resetGridCache keeps tests independent — the grid and anchor caches are global +// so a live stream learns its encoder phase only once. +func resetGridCache(channel string) { + gridMu.Lock() + delete(gridCache, channel) + gridMu.Unlock() + anchorMu.Lock() + delete(anchors, channel) + anchorMu.Unlock() +} + +func TestBuildSyntheticShape(t *testing.T) { + resetGridCache("vctest") + base := originServer(t) + out, err := BuildSynthetic(SyntheticConfig{}, "vctest", base, "df163382-1ddd-fdd5-bec9-822c1ec0f052") + if err != nil { + t.Fatal(err) + } + got := string(out) + + // The default grid is the one the live RTE pipeline was tuned with. + for _, want := range []string{ + `type="dynamic"`, + `minimumUpdatePeriod="PT2S"`, + `minBufferTime="PT8S"`, + `timeShiftBufferDepth="PT1M"`, + `maxSegmentDuration="PT4S"`, + `cenc:default_KID="df163382-1ddd-fdd5-bec9-822c1ec0f052"`, + `timescale="600"`, + `timescale="48000"`, + `bandwidth="6000000"`, + `bandwidth="128000"`, + `codecs="avc1.640028"`, + `codecs="mp4a.40.2"`, + `width="1920" height="1080"`, + `audioSamplingRate="48000"`, + `initialization="vctest-video=6000000.dash"`, + `media="vctest-video=6000000-$Time$.dash"`, + `initialization="vctest-audio_128k=128000.dash"`, + `media="vctest-audio_128k=128000-$Time$.dash"`, + base + "dash/", + } { + if !strings.Contains(got, want) { + t.Errorf("synthetic MPD missing %q\n---\n%s", want, got) + } + } +} + +// The published window must be WindowSegments long: r = count-1. +func TestBuildSyntheticWindowSize(t *testing.T) { + resetGridCache("vcwin") + base := originServer(t) + out, err := BuildSynthetic(SyntheticConfig{WindowSegments: 6}, "vcwin", base, "df1633821dddfdd5bec9822c1ec0f052") + if err != nil { + t.Fatal(err) + } + re := regexp.MustCompile(``) + ms := re.FindAllStringSubmatch(string(out), -1) + if len(ms) != 2 { + t.Fatalf("want 2 SegmentTimelines, got %d", len(ms)) + } + for i, m := range ms { + r, _ := strconv.Atoi(m[3]) + if r != 5 { + t.Errorf("timeline %d: r=%d, want 5 (6 segments)", i, r) + } + d, _ := strconv.Atoi(m[2]) + // 4s segments: 4*600=2400 video ticks, 4*48000=192000 audio ticks. + want := []int{2400, 192000}[i] + if d != want { + t.Errorf("timeline %d: d=%d, want %d", i, d, want) + } + } +} + +// Segment times must land on the encoder's grid, i.e. share its phase offset, +// or the downloader requests segments the origin does not have. +func TestBuildSyntheticAlignsToEncoderPhase(t *testing.T) { + resetGridCache("vcphase") + base := originServer(t) + grid, err := LearnGrid("vcphase", base, SyntheticConfig{}.WithDefaults().GridSpec()) + if err != nil { + t.Fatal(err) + } + out, err := BuildSynthetic(SyntheticConfig{}, "vcphase", base, "df1633821dddfdd5bec9822c1ec0f052") + if err != nil { + t.Fatal(err) + } + re := regexp.MustCompile(``) + ms := re.FindAllStringSubmatch(string(out), -1) + vStart, _ := strconv.ParseInt(ms[0][1], 10, 64) + if got := ((vStart-grid.VMod)%grid.VDur + grid.VDur) % grid.VDur; got != 0 { + t.Errorf("video start %d is off-grid by %d (dur=%d mod=%d)", vStart, got, grid.VDur, grid.VMod) + } + aStart, _ := strconv.ParseInt(ms[1][1], 10, 64) + if got := ((aStart-grid.AMod)%grid.ADur + grid.ADur) % grid.ADur; got != 0 { + t.Errorf("audio start %d is off-grid by %d (dur=%d mod=%d)", aStart, got, grid.ADur, grid.AMod) + } +} + +// Every provider-shaped literal must be overridable from config. +func TestBuildSyntheticHonoursConfig(t *testing.T) { + resetGridCache("ch9") + base := originServer(t) + cfg := SyntheticConfig{ + VideoName: "video", + AudioName: "audio_128k", + VideoBandwidth: 3_000_000, + AudioBandwidth: 96_000, + VideoCodecs: "hvc1.1.6.L93.B0", + AudioCodecs: "mp4a.40.5", + Width: 1280, + Height: 720, + Lang: "ga", + SegmentPathSuffix: "cmaf/", + VideoInitTemplate: "{channel}-v-init.m4s", + VideoMediaTemplate: "{channel}-v-$Time$.m4s", + MinUpdatePeriod: "PT4S", + } + out, err := BuildSynthetic(cfg, "ch9", base, "df1633821dddfdd5bec9822c1ec0f052") + if err != nil { + t.Fatal(err) + } + got := string(out) + for _, want := range []string{ + `bandwidth="3000000"`, `bandwidth="96000"`, + `codecs="hvc1.1.6.L93.B0"`, `codecs="mp4a.40.5"`, + `width="1280" height="720"`, `lang="ga"`, + `minimumUpdatePeriod="PT4S"`, + base + "cmaf/", + `initialization="ch9-v-init.m4s"`, + `media="ch9-v-$Time$.m4s"`, + // audio templates were left empty, so they derive from name + bandwidth + `initialization="ch9-audio_128k=96000.dash"`, + } { + if !strings.Contains(got, want) { + t.Errorf("missing %q\n---\n%s", want, got) + } + } + if strings.Contains(got, `width="1920"`) || strings.Contains(got, `codecs="avc1`) { + t.Error("a default leaked past the config override") + } +} + +func TestBuildSyntheticRejectsBadKID(t *testing.T) { + resetGridCache("vcbad") + base := originServer(t) + if _, err := BuildSynthetic(SyntheticConfig{}, "vcbad", base, "not-a-kid"); err == nil { + t.Fatal("expected an error for a malformed KID") + } +} + +func TestScaleMSSDoesNotOverflow(t *testing.T) { + // A real MSS time (~1.7e16); mss*48000 overflows int64 if done naively. + const mss = int64(17_000_000_000_000_000) + got := ScaleMSS(mss, DefaultMSSTimescale, 48_000) + want := int64(17_000_000_000_000_000 / 10_000_000 * 48_000) + if got != want { + t.Errorf("ScaleMSS = %d, want %d", got, want) + } + if got < 0 { + t.Error("ScaleMSS overflowed to a negative value") + } +} + +func TestParseMSSStreamTimesExpandsRepeats(t *testing.T) { + times := ParseMSSStreamTimes(smoothManifest, "video") + if len(times) != 4 { + t.Fatalf("got %d segment times, want 4 (r=3)", len(times)) + } + for i, ts := range times { + want := int64(17_000_000_000_000) + int64(i)*40_000_000 + if ts != want { + t.Errorf("time[%d] = %d, want %d", i, ts, want) + } + } + if ParseMSSStreamTimes(smoothManifest, "nope") != nil { + t.Error("unknown stream name should yield no times") + } +} + +func TestKIDToUUIDAndHex(t *testing.T) { + u, err := KIDToUUID("DF1633821DDDFDD5BEC9822C1EC0F052") + if err != nil { + t.Fatal(err) + } + if u != "df163382-1ddd-fdd5-bec9-822c1ec0f052" { + t.Errorf("KIDToUUID = %q", u) + } + if KIDHex(u) != "df1633821dddfdd5bec9822c1ec0f052" { + t.Errorf("KIDHex = %q", KIDHex(u)) + } + if _, err := KIDToUUID("abc"); err == nil { + t.Error("short KID should error") + } +} + +func TestExtractDefaultKID(t *testing.T) { + xml := fmt.Sprintf(``, "DF163382-1DDD-FDD5-BEC9-822C1EC0F052") + if got := ExtractDefaultKID(xml); got != "df163382-1ddd-fdd5-bec9-822c1ec0f052" { + t.Errorf("ExtractDefaultKID = %q", got) + } + if ExtractDefaultKID(``) != "" { + t.Error("no KID should yield empty string") + } +} + +// An unregistered or "none" rewriter must report that no rewrite is needed. +func TestRewriterRegistry(t *testing.T) { + if _, needed := Lookup("none"); needed { + t.Error(`"none" must not need a rewrite`) + } + if _, needed := Lookup(""); needed { + t.Error("empty name must not need a rewrite") + } + if _, needed := Lookup("nosuchprovider"); needed { + t.Error("unknown name must not need a rewrite") + } + Register("testrw", func() Rewriter { return Passthrough{} }) + // Registering a Passthrough still means "no rewrite needed". + if _, needed := Lookup("testrw"); needed { + t.Error("a passthrough rewriter must not report needing a rewrite") + } +} diff --git a/apps/pkg/phonecap/phonecap.go b/apps/pkg/phonecap/phonecap.go new file mode 100644 index 0000000..d89fc47 --- /dev/null +++ b/apps/pkg/phonecap/phonecap.go @@ -0,0 +1,463 @@ +// Package phonecap runs one phone MITM capture: proxy → launch → autoplay → +// wait → wvkey → session → optional force-stop. Used by capture.exe and agent. +package phonecap + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "drmdecryption/adb" + "drmdecryption/app" + "drmdecryption/capture" + "drmdecryption/proxy" + "drmdecryption/repo" + "drmdecryption/session" + "drmdecryption/wvkey" +) + +// DefaultCAHash is the subject hash of the bundled MITM CA, used when an app +// module does not state its own. +const DefaultCAHash = "6c3578b4" + +// Options for a single capture job on one device. +type Options struct { + App app.App + Channel string + Client *adb.Client // serial-scoped when multi-device + Root string + Python string + WVD string // path to .wvd device file (required for key fetch) + UserAgent string // optional license-request User-Agent + Wait time.Duration + NoAutoPlay bool + // KeyMode: "auto" (default), "modulardrm", or "raw". + KeyMode string + // CloseApp force-stops the package when the job ends (success or failure). + CloseApp bool + // ClearProxy clears global http_proxy after the job. + ClearProxy bool +} + +// Result is the captured session plus on-disk path. +type Result struct { + Session session.Session + Path string + Key string + MPD string +} + +// RunPassive starts the MITM only — no app launch / auto-play. Waits for +// whatever DRM/manifest traffic the phone generates, prints it, and writes +// outputs/capture// when anything useful appears. +func RunPassive(opt Options) (res Result, err error) { + if opt.Client == nil { + opt.Client = adb.New() + } + if opt.Root == "" { + opt.Root = repo.Root() + } + if opt.Wait <= 0 { + opt.Wait = 180 * time.Second + } + c := opt.Client + root := opt.Root + + if opt.ClearProxy { + defer ClearProxyLater(c) + } + if err := c.EnsureDevice(); err != nil { + return res, err + } + + proxyBin, caHash := resolveProxy(nil, root) + if err := proxy.PushAndStart(c, proxyBin); err != nil { + return res, err + } + if err := proxy.EnsureHTTPProxy(c, ""); err != nil { + return res, err + } + proxy.ReinjectCA(c, caHash) + + fmt.Println("[*] Passive capture — open any app and start playback on the phone") + fmt.Println(" (no auto-play; Ctrl+C will not save — wait for traffic or raise --wait)") + + hints := capture.DefaultPassiveHints() + _ = os.MkdirAll(filepath.Join(root, ".cache"), 0o755) + serialTag := c.Serial + if serialTag == "" { + serialTag = "default" + } + localLog := filepath.Join(root, ".cache", "appproxy-"+sanitize(serialTag)+".log") + localCap := filepath.Join(root, ".cache", "appproxy-"+sanitize(serialTag)+"_cap.json") + stop := make(chan struct{}) + go capture.MirrorLogLoop(c, hints.RemoteLog, localLog, stop) + defer close(stop) + + fmt.Println("[*] Waiting for any license / PSSH / manifest…") + fmt.Println(" watch:", localLog) + capData, err := capture.Wait(c, hints, localLog, localCap, opt.Wait) + if err != nil { + return res, err + } + licenseURL := capData.Get("license_url") + if strings.TrimSpace(capData.Get("pssh")) == "" && strings.TrimSpace(capData.Get("mpd")) != "" { + if pssh, err := extractHLSPSSH(capData.Get("mpd")); err == nil && pssh != "" { + capData["pssh"] = pssh + } + } + + fmt.Println("[+] Capture:") + for _, k := range []string{"pid", "mpd", "license_url", "auth", "pssh"} { + v := capData.Get(k) + if v == "" { + continue + } + label := k + if k == "license_url" { + label = "license" + } + fmt.Println(" ", label+":", trim(v, 110)) + } + + key := "" + if capData.Get("pssh") != "" && licenseURL != "" { + k, kerr := fetchKey(opt, root, capData, licenseURL) + if kerr != nil { + fmt.Fprintf(os.Stderr, "[!] key fetch skipped: %v\n", kerr) + } else { + key = k + fmt.Println("[+] key:", strings.ReplaceAll(key, "\n", " | ")) + } + } else { + fmt.Println("[*] Not enough fields for wvkey (need pssh + license_url) — raw capture saved") + } + + sess := session.Session{ + Channel: opt.Channel, + PSSH: capData.Get("pssh"), + Auth: capData.Get("auth"), + PID: capData.Get("pid"), + Key: key, + MPD: capData.Get("mpd"), + LicenseURL: licenseURL, + } + path, err := session.Write(root, "capture", sess) + if err != nil { + return res, err + } + fmt.Println("[+] session:", path) + res.Session = sess + res.Path = path + res.Key = key + res.MPD = sess.MPD + return res, nil +} + +// Run executes the full phone capture pipeline for a registered app module. +func Run(opt Options) (res Result, err error) { + if opt.App == nil { + return res, fmt.Errorf("app plugin required (pass --app, or omit --app for passive capture)") + } + if opt.Client == nil { + opt.Client = adb.New() + } + if opt.Root == "" { + opt.Root = repo.Root() + } + if opt.Wait <= 0 { + opt.Wait = 180 * time.Second + } + c := opt.Client + a := opt.App + root := opt.Root + + // Defers run LIFO: close app first, then clear proxy. + if opt.ClearProxy { + defer ClearProxyLater(c) + } + if opt.CloseApp { + defer func() { + fmt.Printf("[*] Closing %s…\n", a.Package()) + c.ForceStop(a.Package()) + }() + } + + if err := c.EnsureDevice(); err != nil { + return res, err + } + + proxyBin, caHash := resolveProxy(a, root) + hints := a.CaptureHints() + useTProxy := false + if tm, ok := a.(app.TransparentMITM); ok && tm.UseTransparentMITM() { + useTProxy = true + } + + if useTProxy { + // Transparent REDIRECT — keeps phone VPN (BBC UK geo) working. + proxy.ClearHTTPProxy(c) + remoteDir := "/data/local/tmp/capture/" + a.Package() + if err := proxy.StartTransparent(c, proxyBin, a.Package(), "8080", remoteDir, false); err != nil { + return res, err + } + defer proxy.StopTransparent(c) + hints.RemoteCaps = []string{remoteDir + "/cap.json"} + hints.RemoteLog = remoteDir + "/appproxy.log" + fmt.Println("[*] Transparent MITM (VPN OK) — package", a.Package()) + } else { + if err := proxy.PushAndStart(c, proxyBin, hints.Score.ProxyArgs()...); err != nil { + return res, err + } + if err := proxy.EnsureHTTPProxy(c, ""); err != nil { + return res, err + } + proxy.ReinjectCA(c, caHash) + } + + if err := a.Launch(c); err != nil { + fmt.Fprintf(os.Stderr, "[!] launch: %v\n", err) + } + if !opt.NoAutoPlay { + if err := a.AutoPlay(c, opt.Channel); err != nil { + fmt.Fprintf(os.Stderr, "[!] auto-play failed: %v\n", err) + fmt.Println("[*] Open the channel on the phone manually and start playback…") + } + } else { + fmt.Println("[*] Open the channel on the phone and wait for playback…") + } + + _ = os.MkdirAll(filepath.Join(root, ".cache"), 0o755) + serialTag := c.Serial + if serialTag == "" { + serialTag = "default" + } + localLog := filepath.Join(root, ".cache", "appproxy-"+sanitize(serialTag)+".log") + localCap := filepath.Join(root, ".cache", "appproxy-"+sanitize(serialTag)+"_cap.json") + stop := make(chan struct{}) + go capture.MirrorLogLoop(c, hints.RemoteLog, localLog, stop) + defer close(stop) + + need := hints.Require + if len(need) == 0 { + need = []string{"manifest"} + } + fmt.Printf("[*] Waiting for %s…\n", strings.Join(need, " + ")) + fmt.Println(" watch:", localLog) + capData, err := capture.Wait(c, hints, localLog, localCap, opt.Wait) + if err != nil { + return res, err + } + licenseURL := first(capData.Get("license_url"), a.LicenseURL()) + keyMode := strings.ToLower(strings.TrimSpace(opt.KeyMode)) + if keyMode == "" || keyMode == "auto" { + keyMode = strings.ToLower(strings.TrimSpace(a.KeyMode())) + } + // Brightcove/HLS: proxy often captures license + master URL but not PSSH. + // Skip for clear/MPD-only apps (KeyMode none). + if keyMode != "none" && keyMode != "clear" { + if strings.TrimSpace(capData.Get("pssh")) == "" && strings.TrimSpace(capData.Get("mpd")) != "" { + if pssh, err := extractHLSPSSH(capData.Get("mpd")); err == nil && pssh != "" { + capData["pssh"] = pssh + } else if err != nil { + fmt.Fprintf(os.Stderr, "[!] HLS PSSH extract: %v\n", err) + } + } + } + + mpdURL := strings.TrimSpace(capData.Get("mpd")) + // Console: always surface the MPD we will use (BBC clear streams only need this). + fmt.Println() + fmt.Println("========== MPD ==========") + if mpdURL != "" { + fmt.Println(mpdURL) + } else { + fmt.Println("(none)") + } + fmt.Println("=========================") + fmt.Println() + fmt.Println("[+] Capture:") + for _, k := range []string{"pid", "mpd", "license_url", "auth", "pssh"} { + v := capData.Get(k) + if k == "license_url" { + v = first(v, licenseURL) + } + if v == "" { + continue + } + label := k + if k == "license_url" { + label = "license" + } + fmt.Println(" ", label+":", trim(v, 120)) + } + + key := "" + if keyMode == "none" || keyMode == "clear" { + fmt.Println("[*] Key mode none — skipping wvkey (clear / MPD-only capture)") + } else { + var kerr error + key, kerr = fetchKey(opt, root, capData, licenseURL) + if kerr != nil { + return res, kerr + } + fmt.Println("[+] key:", strings.ReplaceAll(key, "\n", " | ")) + } + + sess := session.Session{ + Channel: opt.Channel, + PSSH: capData.Get("pssh"), + Auth: capData.Get("auth"), + PID: capData.Get("pid"), + Key: key, + MPD: mpdURL, + LicenseURL: licenseURL, + } + path, err := session.Write(root, a.Name(), sess) + if err != nil { + return res, err + } + fmt.Println("[+] session:", path) + if mpdURL != "" { + fmt.Println("[+] mpd:", mpdURL) + } + res.Session = sess + res.Path = path + res.Key = key + res.MPD = sess.MPD + return res, nil +} + +// ClearProxyLater clears http_proxy (exported for agent defer helpers). +func ClearProxyLater(c *adb.Client) { + proxy.ClearHTTPProxy(c) +} + +func resolveProxy(a app.App, root string) (bin, caHash string) { + bin = filepath.Join(root, "bin", "proxy-android-arm64") + caHash = DefaultCAHash + if a != nil { + if p := a.ProxyBin(); p != "" { + bin = p + } + if h := a.CAHash(); h != "" { + caHash = h + } + } + if _, err := os.Stat(bin); err != nil { + for _, p := range []string{ + filepath.Join(root, "bin", "proxy-android-arm64"), + filepath.Join(root, "apps", "proxy", "proxy-android-arm64"), + } { + if st, e := os.Stat(p); e == nil && !st.IsDir() { + bin = p + break + } + } + } + return bin, caHash +} + +func fetchKey(opt Options, root string, cap capture.Data, licenseURL string) (string, error) { + if strings.TrimSpace(opt.WVD) == "" { + return "", fmt.Errorf("wvkey requires --wvd (path to .wvd device file)") + } + py := opt.Python + if py == "" { + py = filepath.Join(root, ".venv", "Scripts", "python.exe") + if _, err := os.Stat(py); err != nil { + py = filepath.Join(root, ".venv", "bin", "python") + } + } + wopt := wvkey.Options{ + Python: py, + Script: filepath.Join(root, "apps", "wvkey", "wvkey.py"), + WVD: opt.WVD, + PSSH: cap.Get("pssh"), + Auth: cap.Get("auth"), + PID: cap.Get("pid"), + LicenseURL: licenseURL, + UserAgent: opt.UserAgent, + } + mode := strings.ToLower(strings.TrimSpace(opt.KeyMode)) + if mode == "" || mode == "auto" { + mode = "modulardrm" + if opt.App != nil { + if m := strings.ToLower(strings.TrimSpace(opt.App.KeyMode())); m != "" { + mode = m + } + } + } + switch mode { + case "raw": + // Brightcove returns multiple CONTENT keys; NRE needs all of them. + keys, err := wvkey.FetchRawAll(wopt) + if err != nil { + return "", err + } + return strings.Join(keys, "\n"), nil + default: + return wvkey.Fetch(wopt) + } +} + +func first(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} + +var ( + reWVKeyURI = regexp.MustCompile(`(?is)KEYFORMAT="urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed".*?URI="data:text/plain;base64,([A-Za-z0-9+/=]+)"`) + reWVKeyURI2 = regexp.MustCompile(`(?is)URI="data:text/plain;base64,([A-Za-z0-9+/=]+)".*?KEYFORMAT="urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed"`) +) + +func extractHLSPSSH(masterURL string) (string, error) { + resp, err := http.Get(masterURL) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return "", fmt.Errorf("HTTP %d", resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + if err != nil { + return "", err + } + s := string(body) + if m := reWVKeyURI.FindStringSubmatch(s); len(m) == 2 { + return m[1], nil + } + if m := reWVKeyURI2.FindStringSubmatch(s); len(m) == 2 { + return m[1], nil + } + return "", fmt.Errorf("no Widevine PSSH in HLS master") +} + +func trim(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} + +func sanitize(s string) string { + s = strings.Map(func(r rune) rune { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + return r + default: + return '_' + } + }, s) + return s +} diff --git a/apps/pkg/proxy/proxy.go b/apps/pkg/proxy/proxy.go new file mode 100644 index 0000000..6117aa5 --- /dev/null +++ b/apps/pkg/proxy/proxy.go @@ -0,0 +1,577 @@ +package proxy + +import ( + "crypto/md5" + "crypto/x509" + "encoding/binary" + "encoding/pem" + "fmt" + "net" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "drmdecryption/adb" + "drmdecryption/repo" +) + +const ( + // Universal on-device paths (still kill legacy rteproxy process names). + RemoteBin = "/data/local/tmp/appproxy" + RemoteCap = "/data/local/tmp/appproxy_cap.json" + RemoteLog = "/data/local/tmp/appproxy.log" + RemoteTraffic = "/data/local/tmp/appproxy_traffic.jsonl" + RemoteCACrt = "/data/local/tmp/rteproxy-ca.crt" + DefaultCAHash = "6c3578b4" +) + +var reWLANIPv4 = regexp.MustCompile(`(?m)^\s*inet\s+(\d{1,3}(?:\.\d{1,3}){3})/`) + +// DeviceWLANIPv4 returns the phone's current wlan0 IPv4 address. +func DeviceWLANIPv4(c *adb.Client) (string, error) { + out := c.Out("shell", "ip", "-f", "inet", "addr", "show", "wlan0") + if m := reWLANIPv4.FindStringSubmatch(out); len(m) == 2 { + return m[1], nil + } + // Fallbacks used on some OEM builds. + for _, prop := range []string{"dhcp.wlan0.ipaddress", "dhcp.eth0.ipaddress"} { + if v := c.Out("shell", "getprop", prop); net.ParseIP(v) != nil && v != "0.0.0.0" { + return v, nil + } + } + return "", fmt.Errorf("no wlan0 IPv4 (is Wi‑Fi connected?)") +} + +// DeviceHTTPProxyAddr returns ":port" for the on-device MITM. +// Loopback (127.0.0.1) must not be used: after MITM, some clients rewrite the +// upstream Host to the proxy address, and appproxy then dials 127.0.0.1:443. +func DeviceHTTPProxyAddr(c *adb.Client, port string) (string, error) { + if strings.TrimSpace(port) == "" { + port = "8080" + } + ip, err := DeviceWLANIPv4(c) + if err != nil { + return "", err + } + return net.JoinHostPort(ip, port), nil +} + +func looksLikeLoopbackProxy(want string) bool { + host, _, err := net.SplitHostPort(strings.TrimSpace(want)) + if err != nil { + host = strings.TrimSpace(want) + } + host = strings.Trim(host, "[]") + return host == "127.0.0.1" || host == "localhost" || host == "::1" || host == "0.0.0.0" || host == "" +} + +// EnsureHTTPProxy points the device at the on-device mitm. +// Empty or loopback want is rewritten to the phone's WLAN IP:8080 so upstream +// MITM dials keep the real destination host (BBC/RTE/etc.). +func EnsureHTTPProxy(c *adb.Client, want string) error { + if looksLikeLoopbackProxy(want) { + port := "8080" + if hostport := strings.TrimSpace(want); hostport != "" { + if _, p, err := net.SplitHostPort(hostport); err == nil && p != "" { + port = p + } + } + addr, err := DeviceHTTPProxyAddr(c, port) + if err != nil { + return err + } + want = addr + } + cur := c.Out("shell", "settings", "get", "global", "http_proxy") + if cur == want { + fmt.Println("[+] HTTP proxy already", want) + return nil + } + fmt.Printf("[*] Setting HTTP proxy -> %s (was %q)\n", want, cur) + _, err := c.Shell("settings", "put", "global", "http_proxy", want) + got := c.Out("shell", "settings", "get", "global", "http_proxy") + if got != want { + return fmt.Errorf("failed to set http_proxy (got %q)", got) + } + return err +} + +// ClearHTTPProxy disables the global HTTP proxy and stops any leftover mitm. +// Always call this after a capture/agent job so the phone can play apps normally. +func ClearHTTPProxy(c *adb.Client) { + _, _ = c.Shell("settings", "put", "global", "http_proxy", ":0") + _, _ = c.Shell("settings", "delete", "global", "http_proxy") + _, _ = c.Shell("settings", "delete", "global", "global_http_proxy_host") + _, _ = c.Shell("settings", "delete", "global", "global_http_proxy_port") + _, _ = c.Shell("settings", "delete", "global", "global_http_proxy_exclusion_list") + stopProxy(c) + fmt.Println("[*] HTTP proxy cleared") +} + +func stopProxy(c *adb.Client) { + script := ` +for name in appproxy rteproxy; do + pid=$(pidof $name 2>/dev/null || true) + if [ -n "$pid" ]; then kill $pid >/dev/null 2>&1 || true; fi +done +sleep 0.5 +for name in appproxy rteproxy; do + pid=$(pidof $name 2>/dev/null || true) + if [ -n "$pid" ]; then kill -9 $pid >/dev/null 2>&1 || true; fi +done +# Root fallback — some builds ignore non-root kill. +if command -v su >/dev/null 2>&1; then + su -c 'killall appproxy rteproxy 2>/dev/null; killall -9 appproxy rteproxy 2>/dev/null; true' 2>/dev/null || true +fi +sleep 0.3 +(pidof appproxy || pidof rteproxy) >/dev/null 2>&1 && echo STILL || echo STOPPED` + out, _ := c.Shell("sh", "-c", script) + if strings.Contains(out, "STILL") { + fmt.Println("[!] old appproxy still running after stop — port 8080 may stay busy") + } +} + +// extraFlags renders extra device flags, quoting each value. +func extraFlags(args []string) string { + if len(args) == 0 { + return "" + } + var b strings.Builder + for _, a := range args { + b.WriteString(" ") + if strings.HasPrefix(a, "-") { + b.WriteString(a) + continue + } + b.WriteString("'" + strings.ReplaceAll(a, "'", "") + "'") + } + return b.String() +} + +// PushAndStart installs and launches the on-device mitm binary. extraArgs are +// appended to its command line — app modules pass their manifest-scoring hosts +// that way, since the device binary cannot read a module's values file. +func PushAndStart(c *adb.Client, localBin string, extraArgs ...string) error { + if st, err := os.Stat(localBin); err != nil || st.IsDir() { + return fmt.Errorf("missing proxy binary %s — build apps/proxy first (proxyctl build)", localBin) + } + fmt.Println("[*] Stopping any old appproxy/rteproxy...") + stopProxy(c) + + fmt.Println("[*] Pushing appproxy...") + if err := c.Push(localBin, RemoteBin); err != nil { + return err + } + _, _ = c.Shell("chmod", "755", RemoteBin) + _, _ = c.Shell("rm", "-f", RemoteCap, "/data/local/tmp/rte_cap.json", "/sdcard/Download/rte_cap.json", "/storage/emulated/0/Download/rte_cap.json", RemoteLog, "/data/local/tmp/rteproxy.log") + + starter := "#!/system/bin/sh\n" + + "exec " + RemoteBin + + " -listen :8080" + + " -out " + RemoteCap + + " -ca-dir /data/local/tmp" + + " -dns 1.1.1.1,1.0.0.1,8.8.8.8,192.168.1.1" + + " >>" + RemoteLog + " 2>&1\n" + + tmp := filepath.Join(os.TempDir(), "start_appproxy.sh") + if err := os.WriteFile(tmp, []byte(starter), 0o755); err != nil { + return err + } + defer os.Remove(tmp) + if err := c.Push(tmp, "/data/local/tmp/start_appproxy.sh"); err != nil { + return err + } + _, _ = c.Shell("chmod", "755", "/data/local/tmp/start_appproxy.sh") + + // Keep backgrounded after adb exits. + _, _, _ = c.Run("shell", "setsid /data/local/tmp/start_appproxy.sh /dev/null 2>&1 &") + + var pid, logHead string + for i := 0; i < 10; i++ { + time.Sleep(400 * time.Millisecond) + pid = c.Out("shell", "pidof", "appproxy") + if pid == "" { + pid = c.Out("shell", "pidof", "rteproxy") + } + logHead = c.Out("shell", "head", "-12", RemoteLog) + if pid != "" && strings.Contains(logHead, "listening") { + break + } + } + if logHead != "" { + fmt.Println(logHead) + } + errLog := c.Out("shell", "cat", RemoteLog) + if strings.Contains(errLog, "address already in use") || + (strings.Contains(errLog, "listen ") && strings.Contains(errLog, "bind:")) { + fmt.Fprintln(os.Stderr, errLog) + return fmt.Errorf("appproxy failed to bind :8080 (old process still holding the port?)") + } + if pid == "" || !strings.Contains(logHead, "listening") { + if errLog != "" { + fmt.Fprintln(os.Stderr, errLog) + } + return fmt.Errorf("appproxy failed to start") + } + fmt.Printf("[+] appproxy pid=%s; capture -> %s\n", pid, RemoteCap) + return nil +} + +// FindLocalBin returns the first existing proxy binary under the repo. +func FindLocalBin(root string) string { + if root == "" { + root = repo.Root() + } + for _, p := range []string{ + filepath.Join(root, "bin", "proxy-android-arm64"), + filepath.Join(root, "apps", "proxy", "proxy-android-arm64"), + filepath.Join(root, "apps", "proxy", "rteproxy-android-arm64"), + filepath.Join(root, "bin", "rteproxy-android-arm64"), + } { + if st, err := os.Stat(p); err == nil && !st.IsDir() { + return p + } + } + return "" +} + +// FindLocalCA returns the first existing MITM CA cert under the repo. +func FindLocalCA(root string) string { + if root == "" { + root = repo.Root() + } + for _, p := range []string{ + filepath.Join(root, "apps", "proxy", "rteproxy-ca.crt"), + filepath.Join(root, "data", "proxy-ca.crt"), + } { + if st, err := os.Stat(p); err == nil && !st.IsDir() { + return p + } + } + return "" +} + +// AndroidCAHash returns the OpenSSL subject_hash_old used for system CA files. +func AndroidCAHash(certPEM []byte) (string, error) { + block, _ := pem.Decode(certPEM) + if block == nil { + return "", fmt.Errorf("no PEM certificate found") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return "", err + } + sum := md5.Sum(cert.RawSubject) + n := binary.LittleEndian.Uint32(sum[0:4]) + return fmt.Sprintf("%08x", n), nil +} + +// InstallCA pushes the MITM CA onto the device as HASH.0 and optionally Magisk-reinjects it. +// When reinject is true, runs the full Magisk conscrypt bind (needs root / Magisk busybox). +func InstallCA(c *adb.Client, localCA string, reinject bool) (hash string, err error) { + if localCA == "" { + localCA = FindLocalCA("") + } + if localCA == "" { + return "", fmt.Errorf("no local CA cert found (expected apps/proxy/rteproxy-ca.crt)") + } + pemBytes, err := os.ReadFile(localCA) + if err != nil { + return "", err + } + hash, err = AndroidCAHash(pemBytes) + if err != nil { + return "", err + } + fmt.Printf("[*] Installing CA %s (hash %s)\n", localCA, hash) + + if err := c.Push(localCA, RemoteCACrt); err != nil { + return hash, err + } + // Also drop a copy named after the process for clarity. + _ = c.Push(localCA, "/data/local/tmp/appproxy-ca.crt") + + tmpHash := filepath.Join(os.TempDir(), hash+".0") + if err := os.WriteFile(tmpHash, pemBytes, 0o644); err != nil { + return hash, err + } + defer os.Remove(tmpHash) + remoteHash := "/data/local/tmp/" + hash + ".0" + if err := c.Push(tmpHash, remoteHash); err != nil { + return hash, err + } + _, _ = c.Shell("chmod", "644", remoteHash, RemoteCACrt) + fmt.Printf("[+] Pushed %s and %s\n", RemoteCACrt, remoteHash) + + if reinject { + ReinjectCA(c, hash) + } else { + fmt.Println("[*] Skipped Magisk reinject (pass -reinject / install-ca --reinject)") + fmt.Println(" User-CA path: Settings → Security → Install a certificate → CA certificate") + } + return hash, nil +} + +// Stop stops the on-device mitm process. +func Stop(c *adb.Client) { + stopProxy(c) +} + +const ( + RemoteTProxyScript = "/data/local/tmp/tproxy_iptables.sh" + RemoteCaptureRoot = "/data/local/tmp/capture" +) + +// StartTransparent pushes appproxy in -transparent mode (no Wi‑Fi http_proxy), +// installs iptables UID REDIRECT for pkg, and writes captures under remoteDir. +func StartTransparent(c *adb.Client, localBin, pkg, port, remoteDir string, reinject bool) error { + if port == "" { + port = "8080" + } + if remoteDir == "" { + remoteDir = RemoteCaptureRoot + "/" + pkg + } + stopProxy(c) + _ = stopTransparentRules(c) + + // Ensure no global HTTP proxy — transparent mode must not use one. + ClearHTTPProxy(c) + + if reinject { + ReinjectCA(c, DefaultCAHash) + } + + if st, err := os.Stat(localBin); err != nil || st.IsDir() { + return fmt.Errorf("missing proxy binary %s", localBin) + } + fmt.Println("[*] Pushing appproxy (transparent)...") + if err := c.Push(localBin, RemoteBin); err != nil { + return err + } + _, _ = c.Shell("chmod", "755", RemoteBin) + + scriptLocal := filepath.Join(repo.Root(), "apps", "proxy", "device", "tproxy_iptables.sh") + if _, err := os.Stat(scriptLocal); err != nil { + return fmt.Errorf("missing %s", scriptLocal) + } + if err := c.Push(scriptLocal, RemoteTProxyScript); err != nil { + return err + } + _, _ = c.Shell("chmod", "755", RemoteTProxyScript) + _, _ = c.Shell("mkdir", "-p", remoteDir) + _, _ = c.Shell("rm", "-f", remoteDir+"/cap.json", remoteDir+"/traffic.jsonl", remoteDir+"/appproxy.log") + + starter := "#!/system/bin/sh\n" + + "mkdir -p '" + remoteDir + "'\n" + + "exec " + RemoteBin + + " -transparent" + + " -listen :" + port + + " -mitm-internal 127.0.0.1:18080" + + " -out-dir '" + remoteDir + "'" + + " -ca-dir /data/local/tmp" + + " -dns 1.1.1.1,1.0.0.1,8.8.8.8" + + " -log-all" + + " -v" + + "\n" + tmp := filepath.Join(os.TempDir(), "start_appproxy_tproxy.sh") + if err := os.WriteFile(tmp, []byte(starter), 0o755); err != nil { + return err + } + defer os.Remove(tmp) + if err := c.Push(tmp, "/data/local/tmp/start_appproxy_tproxy.sh"); err != nil { + return err + } + _, _ = c.Shell("chmod", "755", "/data/local/tmp/start_appproxy_tproxy.sh") + _, _, _ = c.Run("shell", "setsid /data/local/tmp/start_appproxy_tproxy.sh /dev/null 2>&1 &") + + var pid string + for i := 0; i < 15; i++ { + time.Sleep(400 * time.Millisecond) + pid = c.Out("shell", "pidof", "appproxy") + if pid != "" { + break + } + } + if pid == "" { + return fmt.Errorf("appproxy failed to start (transparent)") + } + fmt.Printf("[+] appproxy pid=%s; capture → %s\n", pid, remoteDir) + + out, errOut, err := c.Run("shell", "su", "-c", "sh "+RemoteTProxyScript+" start "+pkg+" "+port) + fmt.Print(out) + if err != nil { + return fmt.Errorf("iptables: %v (%s)", err, strings.TrimSpace(errOut+out)) + } + fmt.Println("[+] iptables REDIRECT installed (UK VPN can stay ON; do not set Wi‑Fi proxy)") + return nil +} + +// StopTransparent removes iptables rules and stops appproxy (does not require Wi‑Fi proxy clear beyond safety). +func StopTransparent(c *adb.Client) { + _ = stopTransparentRules(c) + stopProxy(c) + ClearHTTPProxy(c) + fmt.Println("[*] transparent capture stopped") +} + +func stopTransparentRules(c *adb.Client) error { + out, errOut, err := c.Run("shell", "su", "-c", "sh "+RemoteTProxyScript+" stop") + if strings.TrimSpace(out) != "" { + fmt.Print(out) + } + if err != nil && !strings.Contains(errOut+out, "STOPPED") { + return fmt.Errorf("iptables stop: %v %s", err, errOut) + } + return nil +} + +// PullDir pulls regular files under remoteDir into destDir. +// Avoids `adb shell sh -c "ls …"` — on Windows that often lists `/` instead of the path. +func PullDir(c *adb.Client, remoteDir, destDir string) error { + if err := os.MkdirAll(destDir, 0o755); err != nil { + return err + } + remoteDir = strings.TrimRight(strings.TrimSpace(remoteDir), "/") + // Prefer known capture artifacts; fall back to find -type f. + // Do not use `adb shell sh -c "ls …"` — on Windows that often lists `/`. + var names []string + for _, n := range []string{"cap.json", "traffic.jsonl", "appproxy.log"} { + if fileExistsOnDevice(c, remoteDir+"/"+n) { + names = append(names, n) + } + } + if len(names) == 0 { + list := c.Out("shell", "find", remoteDir, "-maxdepth", "1", "-type", "f") + for _, line := range strings.Split(list, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + names = append(names, filepath.Base(filepath.Clean(line))) + } + } + if len(names) == 0 { + return fmt.Errorf("no files in %s", remoteDir) + } + seen := map[string]bool{} + pulled := 0 + for _, name := range names { + name = strings.TrimSpace(name) + if name == "" || name == "." || name == ".." || strings.ContainsAny(name, `/\`) || seen[name] { + continue + } + seen[name] = true + remote := remoteDir + "/" + name + local := filepath.Join(destDir, name) + if err := c.Pull(remote, local); err != nil { + fmt.Printf("[!] pull %s: %v\n", remote, err) + continue + } + fmt.Printf("[+] %s\n", local) + pulled++ + } + if pulled == 0 { + return fmt.Errorf("failed to pull any files from %s", remoteDir) + } + return nil +} + +func fileExistsOnDevice(c *adb.Client, remote string) bool { + _, _, err := c.Run("shell", "ls", remote) + return err == nil +} + +// PullCaptures pulls traffic/cap/log into destDir (created if missing). +func PullCaptures(c *adb.Client, destDir string) error { + if err := os.MkdirAll(destDir, 0o755); err != nil { + return err + } + for _, remote := range []string{RemoteTraffic, RemoteCap, RemoteLog} { + base := filepath.Base(remote) + local := filepath.Join(destDir, base) + if err := c.Pull(remote, local); err != nil { + fmt.Printf("[!] pull %s: %v\n", remote, err) + continue + } + fmt.Printf("[+] %s\n", local) + } + return nil +} + +// DiscoverOutDir returns outputs/discover/ under the repo root. +func DiscoverOutDir(root, stamp string) string { + if root == "" { + root = repo.Root() + } + if stamp == "" { + stamp = time.Now().Format("20060102-150405") + } + return filepath.Join(root, "outputs", "discover", stamp) +} + +// ReinjectCA best-effort Magisk bind of the mitm CA into conscrypt. +func ReinjectCA(c *adb.Client, caHash string) { + if caHash == "" { + caHash = DefaultCAHash + } + // Bound the per-app nsenter loop — wait on hundreds of PIDs can hang forever. + script := fmt.Sprintf(` +BB=/data/adb/magisk/busybox +HASH=%s +[ -f /data/local/tmp/$HASH.0 ] || { echo CA_SKIP; exit 0; } +[ -x "$BB" ] || { echo CA_SKIP; exit 0; } +rm -rf /data/local/tmp/cacerts-overlay +mkdir -p /data/local/tmp/cacerts-overlay +cp /apex/com.android.conscrypt/cacerts/* /data/local/tmp/cacerts-overlay/ 2>/dev/null || true +cp /data/local/tmp/$HASH.0 /data/local/tmp/cacerts-overlay/$HASH.0 +chmod 644 /data/local/tmp/cacerts-overlay/* +$BB mount -t tmpfs tmpfs /system/etc/security/cacerts 2>/dev/null || true +cp /data/local/tmp/cacerts-overlay/* /system/etc/security/cacerts/ 2>/dev/null || true +chmod 644 /system/etc/security/cacerts/* 2>/dev/null || true +chcon u:object_r:system_file:s0 /system/etc/security/cacerts/* 2>/dev/null || true +$BB mount --bind /system/etc/security/cacerts /apex/com.android.conscrypt/cacerts 2>/dev/null || true +for Z in $(pidof zygote64 2>/dev/null); do + nsenter --mount=/proc/$Z/ns/mnt -- $BB mount --bind /system/etc/security/cacerts /apex/com.android.conscrypt/cacerts 2>/dev/null || true +done +# Only rebind the target app if set; otherwise skip the full zygote-child sweep (hangs). +PKG_UID_FILE=/data/local/tmp/reinject_target_uid +if [ -f "$PKG_UID_FILE" ]; then + TUID=$(cat "$PKG_UID_FILE") + for PID in $(ps -A -o PID=,UID= 2>/dev/null | awk -v u="$TUID" '$2==u {print $1}'); do + nsenter --mount=/proc/$PID/ns/mnt -- $BB mount --bind /system/etc/security/cacerts /apex/com.android.conscrypt/cacerts 2>/dev/null || true + done +fi +ls /apex/com.android.conscrypt/cacerts/$HASH.0 2>/dev/null && echo CA_OK || echo CA_SKIP +`, caHash) + fmt.Println("[*] Re-injecting system CA (Magisk)...") + tmp := filepath.Join(os.TempDir(), "reinject_ca.sh") + _ = os.WriteFile(tmp, []byte("#!/system/bin/sh\n"+script), 0o755) + defer os.Remove(tmp) + _ = c.Push(tmp, "/data/local/tmp/reinject_ca.sh") + _, _ = c.Shell("chmod", "755", "/data/local/tmp/reinject_ca.sh") + // Host-side timeout: su -mm + nsenter has hung on some Magisk builds. + type runResult struct { + out string + } + ch := make(chan runResult, 1) + go func() { + out, _, _ := c.Run("shell", "su", "-mm", "-c", "sh /data/local/tmp/reinject_ca.sh") + ch <- runResult{out: out} + }() + var out string + select { + case r := <-ch: + out = r.out + case <-time.After(20 * time.Second): + fmt.Println("[!] CA reinject timed out after 20s — continuing (CA likely already mounted)") + _, _ = c.Shell("su", "-c", "killall reinject_ca.sh 2>/dev/null; true") + return + } + if strings.Contains(out, "CA_OK") { + fmt.Println("[+] System CA present in conscrypt") + } else { + fmt.Println("[!] CA inject skipped/failed — if TLS errors, re-run Magisk CA mount") + } +} diff --git a/apps/pkg/repo/root.go b/apps/pkg/repo/root.go new file mode 100644 index 0000000..6dae3ca --- /dev/null +++ b/apps/pkg/repo/root.go @@ -0,0 +1,42 @@ +package repo + +import ( + "os" + "path/filepath" +) + +// Root finds the repository root (directory that contains apps/pkg/go.mod +// and is not itself under apps/ — i.e. the real checkout root). +func Root() string { + wd, err := os.Getwd() + if err != nil { + wd = "." + } + cur := wd + for i := 0; i < 16; i++ { + // Require apps/pkg/go.mod AND a sibling marker so that walking from + // inside apps/pkg does not treat apps/ as the root (apps/pkg/go.mod + // would match the old top-level pkg/ layout check). + pkgMod := filepath.Join(cur, "apps", "pkg", "go.mod") + if _, err := os.Stat(pkgMod); err == nil { + if markerOK(cur) { + return cur + } + } + parent := filepath.Dir(cur) + if parent == cur { + break + } + cur = parent + } + return wd +} + +func markerOK(root string) bool { + for _, name := range []string{"build.ps1", "README.md", "configs", "outputs"} { + if _, err := os.Stat(filepath.Join(root, name)); err == nil { + return true + } + } + return false +} diff --git a/apps/pkg/session/session.go b/apps/pkg/session/session.go new file mode 100644 index 0000000..e5a321c --- /dev/null +++ b/apps/pkg/session/session.go @@ -0,0 +1,101 @@ +package session + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// Session is the on-disk capture result (shared with restream tooling). +type Session struct { + App string `json:"app"` + Channel string `json:"channel,omitempty"` + CapturedAt string `json:"captured_at"` + PSSH string `json:"pssh,omitempty"` + Auth string `json:"auth,omitempty"` + PID string `json:"pid,omitempty"` + Key string `json:"key,omitempty"` + KID string `json:"kid,omitempty"` + KeyHex string `json:"key_hex,omitempty"` + MPD string `json:"mpd,omitempty"` + LicenseURL string `json:"license_url,omitempty"` +} + +// Write creates outputs///session.json (+ field txt files + latest). +func Write(root, app string, s Session) (string, error) { + stamp := time.Now().Format("20060102-150405") + dir := filepath.Join(root, "outputs", app, stamp) + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + s.App = app + if s.CapturedAt == "" { + s.CapturedAt = time.Now().Format(time.RFC3339) + } + if s.Key != "" && s.KID == "" { + parts := strings.SplitN(s.Key, ":", 2) + if len(parts) == 2 { + s.KID = parts[0] + s.KeyHex = parts[1] + } + } + raw, err := json.MarshalIndent(s, "", " ") + if err != nil { + return "", err + } + path := filepath.Join(dir, "session.json") + if err := os.WriteFile(path, append(raw, '\n'), 0o644); err != nil { + return "", err + } + fields := map[string]string{ + "pssh": s.PSSH, + "auth": s.Auth, + "pid": s.PID, + "key": s.Key, + "mpd": s.MPD, + } + for name, val := range fields { + if val == "" { + continue + } + _ = os.WriteFile(filepath.Join(dir, name+".txt"), []byte(val+"\n"), 0o644) + } + latest := filepath.Join(root, "outputs", app, "latest") + _ = os.RemoveAll(latest) + // Best-effort directory copy for Windows (symlinks often need admin). + if err := copyDir(dir, latest); err != nil { + fmt.Fprintf(os.Stderr, "[!] latest link/copy: %v\n", err) + } + return path, nil +} + +func copyDir(src, dst string) error { + if err := os.MkdirAll(dst, 0o755); err != nil { + return err + } + entries, err := os.ReadDir(src) + if err != nil { + return err + } + for _, e := range entries { + in := filepath.Join(src, e.Name()) + out := filepath.Join(dst, e.Name()) + if e.IsDir() { + if err := copyDir(in, out); err != nil { + return err + } + continue + } + b, err := os.ReadFile(in) + if err != nil { + return err + } + if err := os.WriteFile(out, b, 0o644); err != nil { + return err + } + } + return nil +} diff --git a/apps/pkg/session/stream.go b/apps/pkg/session/stream.go new file mode 100644 index 0000000..2d64129 --- /dev/null +++ b/apps/pkg/session/stream.go @@ -0,0 +1,66 @@ +package session + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "time" +) + +// Stream is a streamd-ready description of one playable channel, as produced by +// a provider catalog lookup. MPD holds the manifest URL whatever its type. +type Stream struct { + Name string `json:"name"` + Title string `json:"title"` + App string `json:"app"` + Channel string `json:"channel"` + MPD string `json:"mpd"` + Key string `json:"key"` + Keys []string `json:"keys,omitempty"` + PSSH string `json:"pssh,omitempty"` + PrimaryKID string `json:"primary_kid,omitempty"` + HeadersJSON string `json:"headers_json"` + Rewriter string `json:"rewriter"` + LicenseURL string `json:"license_url,omitempty"` + AccountID string `json:"account_id,omitempty"` + VideoID string `json:"video_id,omitempty"` + PlaybackURL string `json:"playback_url,omitempty"` + ManifestType string `json:"manifest_type,omitempty"` +} + +// WriteStream writes outputs///{session.json,mpd.txt,license.txt,…} +// and refreshes outputs//latest. The app name comes from the caller so no +// provider name is baked in here. +func WriteStream(root, app string, info Stream) (string, error) { + stamp := time.Now().Format("20060102-150405") + dir := filepath.Join(root, "outputs", app, stamp) + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + raw, err := json.MarshalIndent(info, "", " ") + if err != nil { + return "", err + } + if err := os.WriteFile(filepath.Join(dir, "session.json"), append(raw, '\n'), 0o644); err != nil { + return "", err + } + write := func(name, val string) { + if val == "" { + return + } + _ = os.WriteFile(filepath.Join(dir, name), []byte(val+"\n"), 0o644) + } + write("mpd.txt", info.MPD) + write("license.txt", info.LicenseURL) + write("pssh.txt", info.PSSH) + write("key.txt", info.Key) + write("pid.txt", info.VideoID) + if len(info.Keys) > 0 { + write("keys.txt", strings.Join(info.Keys, "\n")) + } + latest := filepath.Join(root, "outputs", app, "latest") + _ = os.RemoveAll(latest) + _ = copyDir(dir, latest) + return dir, nil +} diff --git a/apps/pkg/uiflow/uiflow.go b/apps/pkg/uiflow/uiflow.go new file mode 100644 index 0000000..de8da07 --- /dev/null +++ b/apps/pkg/uiflow/uiflow.go @@ -0,0 +1,360 @@ +// Package uiflow holds the phone-UI primitives app modules drive playback with: +// wait for a node, tap a node, wait for playback, find scrollable live cards. +// It knows nothing about any particular app — every threshold and pattern is an +// argument. +package uiflow + +import ( + "fmt" + "regexp" + "strings" + "time" + + "drmdecryption/adb" +) + +// Pick selects among several matching nodes. +type Pick string + +const ( + PickLargest Pick = "largest" + PickSmallest Pick = "smallest" + PickFirst Pick = "first" +) + +// Match describes which UI node to look for. An empty Match never matches. +type Match struct { + TextRE []string + DescRE []string + TextContains []string + DescContains []string + ResourceContains []string + Pick Pick + MinArea int +} + +type compiled struct { + textRE, descRE []*regexp.Regexp + m Match +} + +// CompileRes compiles case-insensitive patterns, skipping empties. +func CompileRes(pats []string) []*regexp.Regexp { + out := make([]*regexp.Regexp, 0, len(pats)) + for _, p := range pats { + if p == "" { + continue + } + out = append(out, regexp.MustCompile("(?i)"+p)) + } + return out +} + +func (m Match) compile() compiled { + return compiled{textRE: CompileRes(m.TextRE), descRE: CompileRes(m.DescRE), m: m} +} + +func (c compiled) empty() bool { + return len(c.textRE) == 0 && len(c.descRE) == 0 && + len(c.m.TextContains) == 0 && len(c.m.DescContains) == 0 && len(c.m.ResourceContains) == 0 +} + +func (c compiled) matches(n adb.Node) bool { + if c.empty() { + return false + } + if n.Text != "" { + for _, p := range c.textRE { + if p.MatchString(n.Text) { + return true + } + } + for _, s := range c.m.TextContains { + if s != "" && strings.Contains(strings.ToLower(n.Text), strings.ToLower(s)) { + return true + } + } + } + if n.Desc != "" { + for _, p := range c.descRE { + if p.MatchString(n.Desc) { + return true + } + } + for _, s := range c.m.DescContains { + if s != "" && strings.Contains(strings.ToLower(n.Desc), strings.ToLower(s)) { + return true + } + } + } + for _, s := range c.m.ResourceContains { + if s != "" && strings.Contains(n.ResourceID, s) { + return true + } + } + return false +} + +func pickNode(hits []adb.Node, pick Pick, minArea int) *adb.Node { + if len(hits) == 0 { + return nil + } + var best *adb.Node + switch pick { + case PickSmallest: + best = &hits[0] + for i := range hits { + if hits[i].Area() < best.Area() { + best = &hits[i] + } + } + case PickFirst: + best = &hits[0] + default: // largest + best = &hits[0] + for i := range hits { + if hits[i].Area() > best.Area() { + best = &hits[i] + } + } + } + if minArea > 0 && best.Area() < minArea { + return nil + } + return best +} + +// Find polls the UI until a node matches, returning it. +func Find(c *adb.Client, cacheDir string, m Match, timeout time.Duration) (*adb.Node, error) { + cm := m.compile() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + nodes, err := c.DumpUI(cacheDir) + if err != nil { + time.Sleep(time.Second) + continue + } + var hits []adb.Node + for _, n := range nodes { + if cm.matches(n) { + hits = append(hits, n) + } + } + if n := pickNode(hits, m.Pick, m.MinArea); n != nil { + return n, nil + } + time.Sleep(time.Second) + } + return nil, fmt.Errorf("ui node not found within %s", timeout) +} + +// WaitUI waits for a node to exist without tapping it. +func WaitUI(c *adb.Client, cacheDir string, m Match, timeout time.Duration) error { + _, err := Find(c, cacheDir, m, timeout) + return err +} + +// TapUI waits for a node and taps its centre. +func TapUI(c *adb.Client, cacheDir string, m Match, timeout time.Duration) error { + n, err := Find(c, cacheDir, m, timeout) + if err != nil { + return err + } + fmt.Printf("[*] tap_ui %q/%q @ %d,%d\n", n.Text, n.Desc, n.CX(), n.CY()) + return c.Tap(n.CX(), n.CY()) +} + +// PlaybackOpts tunes WaitPlayback. The UI signals are a fallback for players +// whose audio session does not show up in dumpsys. +type PlaybackOpts struct { + Timeout time.Duration + SoftFail bool + OrDescContains []string + OrResourceContains []string +} + +// WaitPlayback blocks until the package is playing audio, or a UI signal says the +// player is up. With SoftFail a timeout is logged and tolerated. +func WaitPlayback(c *adb.Client, cacheDir, pkg string, opt PlaybackOpts) error { + if opt.Timeout <= 0 { + opt.Timeout = 45 * time.Second + } + deadline := time.Now().Add(opt.Timeout) + for time.Now().Before(deadline) { + if c.PlaybackActive(pkg) { + fmt.Printf("[+] %s is PLAYING\n", pkg) + return nil + } + if len(opt.OrDescContains) > 0 || len(opt.OrResourceContains) > 0 { + if nodes, err := c.DumpUI(cacheDir); err == nil { + for _, n := range nodes { + desc := strings.ToLower(n.Desc) + for _, d := range opt.OrDescContains { + if d != "" && strings.Contains(desc, strings.ToLower(d)) { + fmt.Printf("[+] UI signal %q\n", d) + return nil + } + } + for _, r := range opt.OrResourceContains { + if r != "" && strings.Contains(n.ResourceID, r) { + fmt.Printf("[+] resource signal %q\n", r) + return nil + } + } + } + } + } + time.Sleep(time.Second) + } + if opt.SoftFail { + fmt.Printf("[!] timed out waiting for playback; continuing capture\n") + return nil + } + return fmt.Errorf("timed out waiting for playback") +} + +// CardOpts describes what a content card looks like in a given app: a clickable +// row at least MinH tall and MinW wide, below MinY, whose description/text is not +// chrome. Callers supply the denylists; nothing is hardcoded. +type CardOpts struct { + MinY, MinH, MinW int + DescDeny []string + TextDeny []string + // ScrollMax limits swipe attempts when the wanted index is off-screen. + ScrollMax int + // Swipe coordinates used to scroll the card list. + SwipeX, SwipeFromY, SwipeToY, SwipeMS int +} + +// WithDefaults fills unset geometry with values that suit a typical 1080x1920 +// phone card list. +func (o CardOpts) WithDefaults() CardOpts { + if o.MinY <= 0 { + o.MinY = 300 + } + if o.MinH <= 0 { + o.MinH = 250 + } + if o.MinW <= 0 { + o.MinW = 600 + } + if o.ScrollMax <= 0 { + o.ScrollMax = 8 + } + if o.SwipeX == 0 { + o.SwipeX = 540 + } + if o.SwipeFromY == 0 { + o.SwipeFromY = 1700 + } + if o.SwipeToY == 0 { + o.SwipeToY = 700 + } + if o.SwipeMS == 0 { + o.SwipeMS = 350 + } + return o +} + +// Cards returns the content cards visible in a UI dump, top to bottom. +func Cards(nodes []adb.Node, opt CardOpts) []adb.Node { + opt = opt.WithDefaults() + var cards []adb.Node + for _, n := range nodes { + if !n.Clickable { + continue + } + h := n.Y2 - n.Y1 + w := n.X2 - n.X1 + if n.Y1 < opt.MinY || h < opt.MinH || w < opt.MinW { + continue + } + if containsAny(n.Desc, opt.DescDeny) { + continue + } + if containsAnyFold(n.Text, opt.TextDeny) { + continue + } + cards = append(cards, n) + } + sortByTop(cards) + return cards +} + +// WaitCard polls for the index-th content card, scrolling when it is not yet on +// screen (each scroll consumes one index, matching how the list advances). +func WaitCard(c *adb.Client, cacheDir string, index int, timeout time.Duration, opt CardOpts) (*adb.Node, error) { + opt = opt.WithDefaults() + deadline := time.Now().Add(timeout) + target := index + var last int + scrolled := 0 + for time.Now().Before(deadline) { + nodes, err := c.DumpUI(cacheDir) + if err == nil { + cards := Cards(nodes, opt) + last = len(cards) + if target >= 0 && target < len(cards) { + card := cards[target] + return &card, nil + } + if target >= last && last > 0 && scrolled < opt.ScrollMax { + fmt.Printf("[*] card need local #%d (%d on screen) — scrolling\n", target, last) + _ = c.Swipe(opt.SwipeX, opt.SwipeFromY, opt.SwipeX, opt.SwipeToY, opt.SwipeMS) + scrolled++ + if target > 0 { + target-- + } + time.Sleep(1200 * time.Millisecond) + continue + } + } + time.Sleep(800 * time.Millisecond) + } + return nil, fmt.Errorf("only saw %d cards (need index %d)", last, index) +} + +// TapCard waits for and taps the index-th content card. +func TapCard(c *adb.Client, cacheDir string, index int, timeout time.Duration, opt CardOpts) error { + card, err := WaitCard(c, cacheDir, index, timeout, opt) + if err != nil { + return err + } + fmt.Printf("[*] tap card #%d @ %d,%d\n", index, card.CX(), card.CY()) + return c.Tap(card.CX(), card.CY()) +} + +// MonkeyLaunch starts an app through its launcher intent. +func MonkeyLaunch(c *adb.Client, pkg string) error { + _, _, err := c.Run("shell", "monkey", "-p", pkg, "-c", "android.intent.category.LAUNCHER", "1") + return err +} + +func containsAny(s string, needles []string) bool { + for _, n := range needles { + if n != "" && strings.Contains(s, n) { + return true + } + } + return false +} + +func containsAnyFold(s string, needles []string) bool { + ls := strings.ToLower(s) + for _, n := range needles { + if n != "" && strings.Contains(ls, strings.ToLower(n)) { + return true + } + } + return false +} + +func sortByTop(cards []adb.Node) { + for i := 0; i < len(cards); i++ { + for j := i + 1; j < len(cards); j++ { + if cards[j].Y1 < cards[i].Y1 { + cards[i], cards[j] = cards[j], cards[i] + } + } + } +} diff --git a/apps/pkg/wvkey/wvkey.go b/apps/pkg/wvkey/wvkey.go new file mode 100644 index 0000000..7516fa2 --- /dev/null +++ b/apps/pkg/wvkey/wvkey.go @@ -0,0 +1,128 @@ +package wvkey + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// Options for the Python wvkey.py helper. +type Options struct { + Python string // e.g. .venv/Scripts/python.exe + Script string // path to wvkey.py + WVD string + PSSH string + Auth string + PID string + LicenseURL string + UserAgent string +} + +// Fetch runs wvkey.py --quiet (ModularDrm) and returns the first KID:KEY line. +func Fetch(opt Options) (string, error) { + lines, err := run(opt, false, false) + if err != nil { + return "", err + } + if len(lines) == 0 { + return "", fmt.Errorf("wvkey produced no KID:KEY") + } + return lines[0], nil +} + +// FetchRaw runs wvkey.py --mode raw (Brightcove / octet-stream license). +func FetchRaw(opt Options) (string, error) { + lines, err := run(opt, true, false) + if err != nil { + return "", err + } + if len(lines) == 0 { + return "", fmt.Errorf("wvkey produced no KID:KEY") + } + return lines[0], nil +} + +// FetchRawAll returns every CONTENT key from a raw license response. +func FetchRawAll(opt Options) ([]string, error) { + return run(opt, true, true) +} + +func run(opt Options, raw, all bool) ([]string, error) { + py := opt.Python + if py == "" { + py = findPython() + } + script := opt.Script + if script == "" { + return nil, fmt.Errorf("wvkey script path required") + } + if strings.TrimSpace(opt.WVD) == "" { + return nil, fmt.Errorf("wvkey --wvd path required") + } + if strings.TrimSpace(opt.PSSH) == "" { + return nil, fmt.Errorf("wvkey --pssh required") + } + if strings.TrimSpace(opt.LicenseURL) == "" { + return nil, fmt.Errorf("wvkey --license-url required") + } + args := []string{ + script, + "--wvd", opt.WVD, + "--pssh", opt.PSSH, + "--license-url", opt.LicenseURL, + "--quiet", + } + if raw { + args = append(args, "--mode", "raw") + } else { + args = append(args, "--mode", "modulardrm", "--auth", opt.Auth, "--pid", opt.PID) + } + if all { + args = append(args, "--all") + } + if opt.UserAgent != "" { + args = append(args, "--user-agent", opt.UserAgent) + } + cmd := exec.Command(py, args...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("wvkey failed: %w\n%s", err, strings.TrimSpace(stderr.String())) + } + var out []string + for _, line := range strings.Split(strings.TrimSpace(stdout.String()), "\n") { + line = strings.TrimSpace(line) + if strings.Count(line, ":") == 1 && len(line) > 40 { + out = append(out, line) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("wvkey produced no KID:KEY (stderr=%s)", strings.TrimSpace(stderr.String())) + } + return out, nil +} + +func findPython() string { + candidates := []string{ + filepath.Join(".venv", "Scripts", "python.exe"), + filepath.Join(".venv", "bin", "python"), + "python3", + "python", + } + for _, c := range candidates { + if filepath.IsAbs(c) || strings.Contains(c, string(os.PathSeparator)) { + if st, err := os.Stat(c); err == nil && !st.IsDir() { + return c + } + continue + } + if p, err := exec.LookPath(c); err == nil { + return p + } + } + return "python" +} diff --git a/apps/proxy/README.md b/apps/proxy/README.md new file mode 100644 index 0000000..38f5e83 --- /dev/null +++ b/apps/proxy/README.md @@ -0,0 +1,102 @@ +# proxy — on-device HTTPS MITM (appproxy) + +Host CLI plus the Android binary that MITMs phone HTTPS, so a capture can see +license URLs, manifests and auth headers. + +```text +apps/proxy/ + proxyctlcmd/ host CLI, hosted by bin/drm as `drm proxy` + device/ linux/arm64 MITM source (module: appproxy) + rteproxy-ca.crt MITM CA (subject hash 6c3578b4) +``` + +The process name on the phone is **`appproxy`**. Some on-disk names still say +`rteproxy-*`, and those paths are still read, so an already-provisioned phone keeps +working. + +## Build + +```bash +go -C apps/cli build -o ../../bin/drm . # host CLI +./bin/drm proxy build # cross-compile the device binary +``` + +`proxy build` compiles `device/` for linux/arm64 into +`apps/proxy/proxy-android-arm64` and copies it to `bin/proxy-android-arm64`. + +## Use + +```bash +# trust the MITM CA (needs Magisk; mounts into the system store) +./bin/drm proxy install-ca --reinject +./bin/drm proxy reinject-ca # mount only, CA already pushed + +# structured capture proxy +./bin/drm proxy start --install-ca --reinject +./bin/drm proxy stop + +# record everything while you explore an unknown app +./bin/drm proxy discover --install-ca --reinject +./bin/drm proxy discover --force-stop # so it inherits the CA mount + +# pull artifacts without re-running, and release the phone +./bin/drm proxy pull +./bin/drm proxy clear-proxy +``` + +Always clear the proxy when done, or the phone keeps pointing at a dead listener. + +### Transparent mode (UK VPN OK — no Wi‑Fi HTTP proxy) + +Root `iptables` redirects only one app’s TCP/443 into on-device `appproxy`. The +phone can stay on NordVPN UK; nothing sets `http_proxy`. + +```bash +# leave UK VPN connected on the phone, then: +./bin/drm proxy transparent --package bbc.iplayer.android --reinject +# open the app / play — Ctrl+C stops iptables and pulls files +``` + +Writes on device (adb-readable): + +```text +/data/local/tmp/capture//cap.json +/data/local/tmp/capture//traffic.jsonl +/data/local/tmp/capture//appproxy.log +``` + +Pull anytime: + +```bash +adb pull /data/local/tmp/capture/bbc.iplayer.android ./bbc-cap +``` + +Force-stop the target app once after CA reinject so it inherits the Magisk CA mount. + +The device binary cannot read an app module's values file, so a module's +manifest-scoring hosts are passed to it as flags (`--match`, `--score-host`, +`--score-deny`). `drm capture` does this automatically. + +## Artifacts + +| File | What | +|---|---| +| `appproxy_traffic.jsonl` / `traffic.jsonl` | every HTTP(S) request/response (discover / transparent) | +| `appproxy_cap.json` / `cap.json` | structured mpd / license / auth / pssh when detected | +| `appproxy.log` | tagged lines: `[MPD]` `[LIC]` `[PSSH]` `[MAN]` `[TPROXY]` | + +Pulled into `outputs/discover//` or `outputs/transparent//`. + +## Device paths + +| Remote | Role | +|---|---| +| `/data/local/tmp/appproxy` | binary | +| `/data/local/tmp/appproxy_cap.json` | structured capture (proxy mode) | +| `/data/local/tmp/capture//` | transparent out-dir (cap + traffic + log) | +| `/data/local/tmp/tproxy_iptables.sh` | UID REDIRECT helper | +| `/data/local/tmp/6c3578b4.0` | system CA hash file | +| Wi‑Fi `http_proxy` | used only in classic proxy mode — **not** in transparent mode | + +**Full guide: [docs/capture.md](../../docs/capture.md)** — CA troubleshooting and how +to mine a discover dump. diff --git a/apps/proxy/device/appproxy.exe b/apps/proxy/device/appproxy.exe new file mode 100644 index 0000000..5b88708 Binary files /dev/null and b/apps/proxy/device/appproxy.exe differ diff --git a/apps/proxy/device/build.sh b/apps/proxy/device/build.sh new file mode 100644 index 0000000..197f901 --- /dev/null +++ b/apps/proxy/device/build.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")" +GO="${GO:-$(command -v go || true)}" +if [[ -z "$GO" && -x /opt/homebrew/bin/go ]]; then + GO=/opt/homebrew/bin/go +fi +if [[ -z "$GO" ]]; then + echo "go not found; brew install go" >&2 + exit 1 +fi +echo "building linux/arm64 with $GO" +OUT="${OUT:-../proxy-android-arm64}" +GOOS=linux GOARCH=arm64 CGO_ENABLED=0 "$GO" build -ldflags='-s -w' -o "$OUT" . +file "$OUT" +ls -lh "$OUT" +# Prefer: from repo root → bin/proxyctl.exe build diff --git a/apps/proxy/device/go.mod b/apps/proxy/device/go.mod new file mode 100644 index 0000000..02d306f --- /dev/null +++ b/apps/proxy/device/go.mod @@ -0,0 +1,14 @@ +module appproxy + +go 1.27.1 + +require ( + github.com/andybalholm/brotli v1.0.6 // indirect + github.com/elazarl/goproxy v1.9.2 // indirect + github.com/klauspost/compress v1.17.4 // indirect + github.com/refraction-networking/utls v1.8.2 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/net v0.50.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.34.0 // indirect +) diff --git a/apps/proxy/device/go.sum b/apps/proxy/device/go.sum new file mode 100644 index 0000000..6860447 --- /dev/null +++ b/apps/proxy/device/go.sum @@ -0,0 +1,16 @@ +github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= +github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/elazarl/goproxy v1.9.2 h1:+vXRRSWrznMtBrAb559qfqC+Cny1Q3rR0l51Yu/3WUw= +github.com/elazarl/goproxy v1.9.2/go.mod h1:THdE5ix2clxX9lZzcICPpZ67d6CdrPZxdOYsNgU5e30= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= +github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= +golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= diff --git a/apps/proxy/device/main.go b/apps/proxy/device/main.go new file mode 100644 index 0000000..248bd8f --- /dev/null +++ b/apps/proxy/device/main.go @@ -0,0 +1,1310 @@ +// appproxy — on-device HTTPS MITM proxy that logs DASH/HLS manifests + DRM +// fields to a JSON file (same shape as watchhttp.py / from_har.py). +// +// Cross-compile (host): +// +// GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -o appproxy-android-arm64 . +// +// On device (example): +// +// adb push appproxy-android-arm64 /data/local/tmp/appproxy +// adb shell chmod 755 /data/local/tmp/appproxy +// adb shell /data/local/tmp/appproxy -listen :8080 -out /data/local/tmp/appproxy_cap.json +// adb pull /data/local/tmp/appproxy-ca.crt # install as CA on the phone +package main + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "flag" + "fmt" + "io" + "log" + "math/big" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/elazarl/goproxy" +) + +type capture struct { + MPD string `json:"mpd,omitempty"` + LicenseURL string `json:"license_url,omitempty"` + Auth string `json:"auth,omitempty"` + PID string `json:"pid,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` + PSSH string `json:"pssh,omitempty"` // first short Widevine PSSH seen in an MPD body +} + +type licenseBody struct { + GetWidevineLicense struct { + ReleasePID string `json:"releasePid"` + WidevineChallenge string `json:"widevineChallenge"` + } `json:"getWidevineLicense"` +} + +func main() { + listen := flag.String("listen", ":8080", "proxy listen address (transparent public port when -transparent)") + outPath := flag.String("out", defaultOutPath(), "JSON capture output path (structured mpd/license)") + outDir := flag.String("out-dir", "", "if set, write cap.json + traffic.jsonl (+ log) into this directory") + caDir := flag.String("ca-dir", defaultCADir(), "directory for CA cert/key (created if missing)") + dnsServers := flag.String("dns", "", "comma-separated DNS servers (default: device DNS or 1.1.1.1)") + verbose := flag.Bool("v", true, "log matched exchanges to stderr") + logAll := flag.Bool("log-all", false, "log every HTTP(S) request (discover mode for unknown apps)") + trafficPath := flag.String("traffic", defaultTrafficPath(), "JSONL traffic log path (used with -log-all)") + transparent := flag.Bool("transparent", false, "iptables-REDIRECT mode: no Wi‑Fi http_proxy; works with UK VPN") + mitmInternal := flag.String("mitm-internal", "127.0.0.1:18080", "internal MITM listen addr used by -transparent") + passAll := flag.Bool("passthrough-all", false, "splice every host without MITM (diagnose redirect/VPN vs MITM breakage)") + // Provider-specific needles are flags, not constants: this binary runs on the + // phone and cannot read an app module's config. proxyctl passes them through. + flag.Var(&matchNeedles, "match", "extra host/URL substring that marks an exchange interesting (repeatable)") + flag.Var(&scoreHosts, "score-host", "host that scores as a known manifest origin (repeatable)") + flag.Var(&scoreDeny, "score-deny", "host/URL substring that is never a manifest (repeatable)") + flag.Var(&mitmHosts, "mitm-host", "force MITM for this host (overrides BBC/CDN passthrough; repeatable)") + flag.Parse() + passthroughAll = *passAll + passthroughDefault = *transparent // transparent: MITM only open.live / -mitm-host + forceMITM = forceMITM[:0] + for _, h := range mitmHosts { + forceMITM = append(forceMITM, strings.ToLower(strings.TrimSpace(h))) + } + + if dir := strings.TrimSpace(*outDir); dir != "" { + if err := os.MkdirAll(dir, 0o755); err != nil { + log.Fatalf("out-dir: %v", err) + } + *outPath = filepath.Join(dir, "cap.json") + *trafficPath = filepath.Join(dir, "traffic.jsonl") + logFile := filepath.Join(dir, "appproxy.log") + f, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + if err == nil { + log.SetOutput(io.MultiWriter(os.Stderr, f)) + } + log.Printf("out-dir → %s", dir) + } + + caCertPath := filepath.Join(*caDir, caCertName) + caKeyPath := filepath.Join(*caDir, caKeyName) + // Older devices were provisioned with the previous CA file names; keep using + // those if they are already on the phone so a reinject is not required. + if _, err := os.Stat(caCertPath); err != nil { + if _, e := os.Stat(filepath.Join(*caDir, legacyCACertName)); e == nil { + caCertPath = filepath.Join(*caDir, legacyCACertName) + caKeyPath = filepath.Join(*caDir, legacyCAKeyName) + } + } + if err := os.MkdirAll(*caDir, 0o755); err != nil { + log.Fatalf("ca-dir: %v", err) + } + tlsCert, err := loadOrCreateCA(caCertPath, caKeyPath) + if err != nil { + log.Fatalf("CA: %v", err) + } + log.Printf("CA cert: %s (install this on the device as a trusted CA)", caCertPath) + log.Printf("capture → %s", *outPath) + if *logAll { + log.Printf("discover → %s (-log-all: every request)", *trafficPath) + _ = os.Remove(*trafficPath) // fresh session + } + + resolver := newResolver(*dnsServers) + log.Printf("DNS → %v", resolver.addrs) + baseDialer := &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + Resolver: resolver.std, + } + // Dial upstream by hostname, but never follow AdAway/hosts sinkholes + // (127.0.0.1) — re-resolve via plain DNS when /etc/hosts poisoned the name. + dialContext := func(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("dial addr %s: %w", addr, err) + } + if ip := net.ParseIP(host); ip != nil { + if ip.IsLoopback() || ip.IsUnspecified() { + return nil, fmt.Errorf("refusing loopback upstream dial %s (Wi‑Fi proxy must be device LAN IP; also disable VPN during capture)", addr) + } + return baseDialer.DialContext(ctx, network, addr) + } + ips, err := resolver.lookupIPs(ctx, host) + if err != nil { + return nil, fmt.Errorf("resolve %s: %w", host, err) + } + var last error + for _, ip := range ips { + if ip.IsLoopback() || ip.IsUnspecified() { + continue + } + target := net.JoinHostPort(ip.String(), port) + c, err := baseDialer.DialContext(ctx, network, target) + if err == nil { + return c, nil + } + last = err + } + if last == nil { + last = fmt.Errorf("all resolved IPs for %s were loopback/unusable (AdAway hosts or VPN fake-ip?)", host) + } + return nil, last + } + // dialPreferOrig uses NordVPN/app SO_ORIGINAL_DST when recorded for this host + // so BBC geolocation sees the same path as the app (fake-IP → UK tunnel). + dialPreferOrig := func(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err == nil { + if v, ok := origDstByHost.Load(strings.ToLower(host)); ok { + od := v.(string) + if od != "" && od != addr { + log.Printf("[DIAL] %s → orig-dst %s", addr, od) + return dialContext(ctx, network, od) + } + } + _ = port + } + return dialContext(ctx, network, addr) + } + + transport := &http.Transport{ + Proxy: nil, // never chain through another proxy / env + DialContext: dialPreferOrig, + // Chrome-like upstream TLS — BBC Fastly often 403s Go's default JA3. + DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + c, err := dialTLSChrome(ctx, dialPreferOrig, network, addr) + if err != nil { + log.Printf("[TLS] utls failed for %s: %v — std fallback", addr, err) + return dialTLSStd(ctx, dialPreferOrig, network, addr) + } + return c, nil + }, + ForceAttemptHTTP2: false, // utls conn is HTTP/1.1-only + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 15 * time.Second, + } + + cap := &captureStore{path: *outPath} + traffic := &trafficLog{path: *trafficPath, enabled: *logAll} + + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = false + proxy.Tr = transport + // goproxy also dials CONNECT targets via its own transport; keep them aligned. + proxy.ConnectDial = func(network, addr string) (net.Conn, error) { + return dialPreferOrig(context.Background(), network, addr) + } + proxy.ConnectDialWithReq = func(req *http.Request, network, addr string) (net.Conn, error) { + // Prefer the CONNECT / request host when addr collapsed to loopback. + if host, port, err := net.SplitHostPort(addr); err == nil && hostIsLoopback(host) { + if rh := requestHost(req); rh != "" && !hostIsLoopback(rh) { + addr = net.JoinHostPort(stripHostPort(rh), port) + log.Printf("[FIX] CONNECT dial %s → %s", host, addr) + } + } + if req != nil { + if od := req.Header.Get("X-Appproxy-Orig-Dst"); od != "" { + rememberOrigDst(addr, od) + } + } + return dialPreferOrig(context.Background(), network, addr) + } + + // MITM all CONNECT tunnels so we can read HTTPS bodies. + customCA := &goproxy.ConnectAction{ + Action: goproxy.ConnectMitm, + TLSConfig: goproxy.TLSConfigFromCA(tlsCert), + } + proxy.OnRequest().HandleConnect(goproxy.FuncHttpsHandler(func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { + ud := map[string]string{"connect_host": host} + if ctx.Req != nil { + if od := ctx.Req.Header.Get("X-Appproxy-Orig-Dst"); od != "" { + ud["orig_dst"] = od + rememberOrigDst(host, od) + log.Printf("[CONNECT] %s (orig-dst %s)", host, od) + } + } + ctx.UserData = ud + if ud["orig_dst"] == "" && (*logAll || interesting(host, host)) { + log.Printf("[CONNECT] %s", host) + } + if hostIsLoopback(stripHostPort(host)) { + log.Printf("[WARN] CONNECT to loopback %s — client likely using 127.0.0.1 as Wi‑Fi proxy", host) + } + return customCA, host + })) + + proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + // After MITM, some Android stacks put the proxy address (127.0.0.1) in + // Host / URL. Restore the CONNECT target so upstream dials the real site. + if ch := connectHostFromCtx(ctx); ch != "" { + fixLoopbackHost(req, ch) + } + + url := req.URL.String() + if req.URL.Scheme == "" { + // After MITM, URL may be relative; rebuild. + scheme := "https" + if req.TLS == nil && ctx.Req != nil && ctx.Req.URL != nil && ctx.Req.URL.Scheme != "" { + scheme = ctx.Req.URL.Scheme + } + host := req.Host + if host == "" && req.URL.Host != "" { + host = req.URL.Host + } + url = scheme + "://" + host + req.URL.RequestURI() + } + + host := req.Host + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + + kind := classifyURL(req.Method, host, url) + if (*logAll || interesting(host, url) || kind != "other") && *verbose { + tag := "REQ" + switch kind { + case "mpd": + tag = "MPD" + case "license": + tag = "LIC" + case "manifest": + tag = "MAN" + } + log.Printf("[%s] %s %s", tag, req.Method, trim(url, 220)) + } + if *logAll { + rec := map[string]any{ + "ts": time.Now().Format(time.RFC3339), + "phase": "req", + "kind": kind, + "method": req.Method, + "host": host, + "url": url, + } + if auth := req.Header.Get("Authorization"); auth != "" && (kind == "license" || kind == "other") { + rec["authorization"] = trim(auth, 120) + } + if ct := req.Header.Get("Content-Type"); ct != "" { + rec["req_content_type"] = ct + } + traffic.write(rec) + } + + // Live / DASH MPD (any host — RTE sometimes redirects via dai.google.com etc.) + if looksLikeMPD(url) { + cap.set(func(c *capture) { c.MPD = preferMPD(c.MPD, url) }) + if *verbose { + log.Printf("[MPD] %s", url) + } + } + + // ModularDrm license (RTE) + generic license-looking POSTs for discover + isLicHost := strings.Contains(host, "widevine.entitlement") || looksLikeLicense(req.Method, host, url) + if isLicHost && req.Method == http.MethodPost { + auth := req.Header.Get("Authorization") + body, err := io.ReadAll(req.Body) + if err == nil { + req.Body.Close() + req.Body = io.NopCloser(strings.NewReader(string(body))) + } + pid := "" + var lb licenseBody + if json.Unmarshal(body, &lb) == nil { + pid = lb.GetWidevineLicense.ReleasePID + } + cap.set(func(c *capture) { + c.LicenseURL = url + if auth != "" { + c.Auth = auth + } + if pid != "" { + c.PID = pid + } + }) + if *verbose { + log.Printf("[LIC] pid=%s auth=%s", pid, trim(auth, 40)) + log.Printf(" %s", trim(url, 120)) + } + if *logAll { + traffic.write(map[string]any{ + "ts": time.Now().Format(time.RFC3339), + "phase": "license", + "kind": "license", + "method": req.Method, + "host": host, + "url": url, + "authorization": trim(auth, 200), + "pid": pid, + "body_len": len(body), + "body_head": trim(string(body), 240), + }) + } + } + return req, nil + }) + + proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + if resp == nil || resp.Request == nil { + return resp + } + url := resp.Request.URL.String() + host := resp.Request.Host + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + ct := resp.Header.Get("Content-Type") + isMPD := looksLikeMPD(url) || + strings.Contains(ct, "dash+xml") || + strings.Contains(ct, "application/mpd") || + strings.Contains(ct, "mpegurl") || + strings.Contains(ct, "m3u8") || + hostIsManifestOrigin(host) + if *logAll { + kind := classifyURL(resp.Request.Method, host, url) + if isMPD { + kind = "mpd" + } + rec := map[string]any{ + "ts": time.Now().Format(time.RFC3339), + "phase": "resp", + "kind": kind, + "method": resp.Request.Method, + "host": host, + "url": url, + "status": resp.StatusCode, + "resp_content_type": ct, + } + if kind != "other" || isMPD { + traffic.write(rec) + } else if *logAll { + // still record non-other; for pure other skip body-heavy spam unless status interesting + traffic.write(rec) + } + } + // Surface mediaselector / license / non-2xx so play failures are obvious in appproxy.log. + if resp.StatusCode >= 400 || strings.Contains(url, "mediaselector") || interesting(host, url) { + log.Printf("[RSP] %d %s %s", resp.StatusCode, resp.Request.Method, trim(url, 180)) + } + + isMediaSelector := strings.Contains(url, "mediaselector") + if (isMediaSelector || isMPD) && resp.Body != nil { + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + resp.Body = io.NopCloser(strings.NewReader("")) + return resp + } + resp.Body = io.NopCloser(strings.NewReader(string(body))) + if isMediaSelector { + captureMediaSelector(body, url, *outPath, cap, traffic, *logAll, *verbose) + } + if isMPD { + if pssh := firstShortPSSH(string(body)); pssh != "" { + reqURL := url + if reqURL == "" || !strings.Contains(reqURL, "://") { + reqURL = "https://" + host + resp.Request.URL.RequestURI() + } + cap.set(func(c *capture) { + c.PSSH = pssh + if looksLikeMPD(reqURL) { + c.MPD = preferMPD(c.MPD, reqURL) + } + }) + if *verbose { + log.Printf("[PSSH] %s", trim(pssh, 80)) + if looksLikeMPD(reqURL) { + log.Printf("[MPD] %s", reqURL) + } + } + if *logAll { + traffic.write(map[string]any{ + "ts": time.Now().Format(time.RFC3339), + "phase": "pssh", + "kind": "mpd", + "host": host, + "url": reqURL, + "pssh": trim(pssh, 200), + "body_len": len(body), + }) + } + } + } + return resp + } + return resp + }) + + if *transparent { + mitmLn, err := net.Listen("tcp", *mitmInternal) + if err != nil { + log.Fatalf("mitm-internal listen %s: %v", *mitmInternal, err) + } + go func() { + log.Printf("MITM (internal) on %s", mitmLn.Addr()) + log.Fatal(http.Serve(mitmLn, proxy)) + }() + pubLn, err := net.Listen("tcp", *listen) + if err != nil { + log.Fatalf("transparent listen %s: %v", *listen, err) + } + log.Printf("transparent on %s — use iptables REDIRECT (no Wi‑Fi http_proxy; VPN OK)", pubLn.Addr()) + serveTransparentAccept(pubLn, *mitmInternal, func(network, addr string) (net.Conn, error) { + return dialContext(context.Background(), network, addr) + }) + return + } + + ln, err := net.Listen("tcp", *listen) + if err != nil { + log.Fatalf("listen %s: %v", *listen, err) + } + log.Printf("listening on %s (set Wi‑Fi HTTP proxy to this device IP%s)", ln.Addr(), portSuffix(*listen)) + log.Fatal(http.Serve(ln, proxy)) +} + +type dnsResolver struct { + addrs []string + std *net.Resolver +} + +func newResolver(flagDNS string) *dnsResolver { + var addrs []string + if flagDNS != "" { + for _, p := range strings.Split(flagDNS, ",") { + p = strings.TrimSpace(p) + if p != "" { + addrs = append(addrs, p) + } + } + } + if len(addrs) == 0 { + addrs = detectDeviceDNS() + } + // Drop VPN/fake DNS (CGNAT 100.64/10, etc.) — prefer public resolvers for MITM upstream. + cleaned := make([]string, 0, len(addrs)) + for _, a := range addrs { + host := a + if h, _, err := net.SplitHostPort(a); err == nil { + host = h + } + if ip := net.ParseIP(host); ip != nil && (ip.IsPrivate() || ip.IsLoopback() || isCGNAT(ip)) { + continue + } + cleaned = append(cleaned, a) + } + addrs = cleaned + if len(addrs) == 0 { + addrs = []string{"1.1.1.1", "1.0.0.1", "8.8.8.8"} + } + // Ensure host:port form for Dial + for i, a := range addrs { + if !strings.Contains(a, ":") { + addrs[i] = net.JoinHostPort(a, "53") + } + } + r := &dnsResolver{addrs: addrs} + r.std = &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + var last error + d := net.Dialer{Timeout: 3 * time.Second} + for _, server := range r.addrs { + conn, err := d.DialContext(ctx, "udp", server) + if err == nil { + return conn, nil + } + last = err + } + return nil, last + }, + } + return r +} + +func isCGNAT(ip net.IP) bool { + ip4 := ip.To4() + if ip4 == nil { + return false + } + // 100.64.0.0/10 used by NordVPN / carrier-grade NAT DNS + return ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127 +} + +// lookupIPs resolves host for upstream dials. Go's resolver honors /etc/hosts +// (AdAway -> 127.0.0.1); when that happens we fall back to a raw DNS A query. +func (r *dnsResolver) lookupIPs(ctx context.Context, host string) ([]net.IP, error) { + addrs, err := r.std.LookupIPAddr(ctx, host) + out := make([]net.IP, 0, len(addrs)) + for _, a := range addrs { + if a.IP.IsLoopback() || a.IP.IsUnspecified() { + continue + } + out = append(out, a.IP) + } + if len(out) > 0 { + return out, nil + } + raw, rawErr := r.lookupIPRaw(ctx, host) + if len(raw) > 0 { + log.Printf("[DNS] %s: ignored hosts sinkhole, using %v", host, raw) + return raw, nil + } + if err != nil { + return nil, err + } + if rawErr != nil { + return nil, rawErr + } + return nil, fmt.Errorf("no usable IPs for %s", host) +} + +func (r *dnsResolver) lookupIPRaw(ctx context.Context, host string) ([]net.IP, error) { + var last error + d := &net.Dialer{Timeout: 3 * time.Second} + for _, server := range r.addrs { + ips, err := dnsQueryA(ctx, d, server, host) + if err != nil { + last = err + continue + } + found := make([]net.IP, 0, len(ips)) + for _, ip := range ips { + if !ip.IsLoopback() && !ip.IsUnspecified() { + found = append(found, ip) + } + } + if len(found) > 0 { + return found, nil + } + } + return nil, last +} + +// dnsQueryA is a minimal DNS A lookup that never reads /etc/hosts. +func dnsQueryA(ctx context.Context, d *net.Dialer, server, host string) ([]net.IP, error) { + conn, err := d.DialContext(ctx, "udp", server) + if err != nil { + return nil, err + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(3 * time.Second)) + + id := uint16(time.Now().UnixNano()) + msg := buildDNSQuery(id, host) + if _, err := conn.Write(msg); err != nil { + return nil, err + } + buf := make([]byte, 512) + n, err := conn.Read(buf) + if err != nil { + return nil, err + } + return parseDNSAnswersA(buf[:n], id) +} + +func buildDNSQuery(id uint16, host string) []byte { + b := make([]byte, 0, 64) + b = append(b, byte(id>>8), byte(id)) + b = append(b, 0x01, 0x00) // RD + b = append(b, 0x00, 0x01) // QDCOUNT + b = append(b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00) + for _, label := range strings.Split(strings.TrimSuffix(host, "."), ".") { + if label == "" { + continue + } + b = append(b, byte(len(label))) + b = append(b, label...) + } + b = append(b, 0x00) + b = append(b, 0x00, 0x01) // TYPE A + b = append(b, 0x00, 0x01) // CLASS IN + return b +} + +func parseDNSAnswersA(msg []byte, wantID uint16) ([]net.IP, error) { + if len(msg) < 12 { + return nil, fmt.Errorf("short DNS response") + } + id := uint16(msg[0])<<8 | uint16(msg[1]) + if id != wantID { + return nil, fmt.Errorf("DNS id mismatch") + } + qd := int(msg[4])<<8 | int(msg[5]) + an := int(msg[6])<<8 | int(msg[7]) + i := 12 + for q := 0; q < qd && i < len(msg); q++ { + for i < len(msg) && msg[i] != 0 { + if msg[i]&0xC0 == 0xC0 { + i += 2 + break + } + i += 1 + int(msg[i]) + } + if i < len(msg) && msg[i] == 0 { + i++ + } + i += 4 + } + var ips []net.IP + for a := 0; a < an && i+10 <= len(msg); a++ { + if msg[i]&0xC0 == 0xC0 { + i += 2 + } else { + for i < len(msg) && msg[i] != 0 { + i += 1 + int(msg[i]) + } + i++ + } + if i+10 > len(msg) { + break + } + typ := int(msg[i])<<8 | int(msg[i+1]) + rdlen := int(msg[i+8])<<8 | int(msg[i+9]) + i += 10 + if i+rdlen > len(msg) { + break + } + if typ == 1 && rdlen == 4 { + ips = append(ips, net.IPv4(msg[i], msg[i+1], msg[i+2], msg[i+3])) + } + i += rdlen + } + if len(ips) == 0 { + return nil, fmt.Errorf("no A records") + } + return ips, nil +} +func detectDeviceDNS() []string { + // Prefer getprop / dumpsys-style values when available. + out, err := exec.Command("getprop").CombinedOutput() + if err == nil { + var found []string + for _, line := range strings.Split(string(out), "\n") { + // [net.dns1]: [1.1.1.1] or dhcp.*.dns* + if !strings.Contains(line, "dns") { + continue + } + start := strings.Index(line, "[") + end := strings.LastIndex(line, "]") + if start < 0 || end <= start { + continue + } + // value is last [...] + rest := line[start+1 : end] + if i := strings.LastIndex(rest, "["); i >= 0 { + rest = rest[i+1:] + } + rest = strings.TrimSpace(rest) + if net.ParseIP(rest) != nil { + found = append(found, rest) + } + } + if len(found) > 0 { + return unique(found) + } + } + return nil +} + +func unique(in []string) []string { + seen := map[string]bool{} + var out []string + for _, s := range in { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + return out +} + +func portSuffix(listen string) string { + _, port, err := net.SplitHostPort(listen) + if err != nil { + if strings.HasPrefix(listen, ":") { + return listen + } + return "" + } + return ":" + port +} + +// origDstByHost maps lowercase hostname → "ip:port" from transparent SO_ORIGINAL_DST +// so upstream dials keep NordVPN fake-IP / geo routing. +var origDstByHost sync.Map + +func rememberOrigDst(hostOrAddr, origDst string) { + h := strings.ToLower(stripHostPort(hostOrAddr)) + if h == "" || origDst == "" { + return + } + origDstByHost.Store(h, origDst) +} + +// captureMediaSelector saves BBC mediaselector JSON and extracts Widevine licence / +// media hosts for -mitm-host follow-up. +func captureMediaSelector(body []byte, url, outPath string, cap *captureStore, traffic *trafficLog, logAll, verbose bool) { + dir := filepath.Dir(outPath) + if dir == "" || dir == "." { + dir = "/data/local/tmp" + } + msPath := filepath.Join(dir, "mediaselector.json") + if err := os.WriteFile(msPath, body, 0o644); err != nil { + log.Printf("[MS] write %s: %v", msPath, err) + } else { + log.Printf("[MS] saved %s (%d bytes)", msPath, len(body)) + } + + licURLs, mediaURLs, hosts := extractBBCMediaSelector(body) + for _, u := range licURLs { + log.Printf("[LIC-URL] %s", trim(u, 200)) + if cap != nil { + cap.set(func(c *capture) { + if c.LicenseURL == "" { + c.LicenseURL = u + } + }) + } + } + for _, u := range mediaURLs { + if verbose || strings.Contains(strings.ToLower(u), "mpd") || strings.Contains(u, ".mpd") { + log.Printf("[MEDIA] %s", trim(u, 200)) + } + if cap != nil && (strings.Contains(u, ".mpd") || strings.Contains(strings.ToLower(u), "manifest")) { + cap.set(func(c *capture) { + c.MPD = preferMPD(c.MPD, u) + }) + } + } + for _, h := range hosts { + log.Printf("[MS-HOST] %s", h) + } + if logAll && traffic != nil { + traffic.write(map[string]any{ + "ts": time.Now().Format(time.RFC3339), + "phase": "mediaselector", + "kind": "mediaselector", + "url": url, + "body_len": len(body), + "license_urls": licURLs, + "media_urls": mediaURLs, + "hosts": hosts, + }) + } +} + +func extractBBCMediaSelector(body []byte) (licURLs, mediaURLs, hosts []string) { + s := string(body) + // Licence / DRM endpoints (Widevine laurl, authorisationService, etc.) + licNeedles := []string{ + `"https://`, `"http://`, + } + seenU := map[string]bool{} + seenH := map[string]bool{} + addURL := func(u string, license bool) { + u = strings.TrimSpace(u) + u = strings.Trim(u, `"'`) + if u == "" || seenU[u] { + return + } + seenU[u] = true + if license { + licURLs = append(licURLs, u) + } else { + mediaURLs = append(mediaURLs, u) + } + if hp, err := parseURLHost(u); err == nil && hp != "" && !seenH[hp] { + seenH[hp] = true + hosts = append(hosts, hp) + } + } + // Walk JSON-ish strings for http(s) URLs. + for i := 0; i < len(s); { + j := strings.Index(s[i:], "https://") + k := strings.Index(s[i:], "http://") + off := -1 + if j >= 0 && (k < 0 || j < k) { + off = i + j + } else if k >= 0 { + off = i + k + } + if off < 0 { + break + } + end := off + for end < len(s) { + c := s[end] + if c == '"' || c == '\'' || c == ' ' || c == '\n' || c == '\r' || c == '<' || c == '>' { + break + } + end++ + } + u := s[off:end] + u = strings.TrimRight(u, `\,)]}`) + low := strings.ToLower(u) + isLic := strings.Contains(low, "license") || strings.Contains(low, "licence") || + strings.Contains(low, "widevine") || strings.Contains(low, "drm") || + strings.Contains(low, "authorisation") || strings.Contains(low, "authorization") || + strings.Contains(low, "/la/") || strings.Contains(low, "laurl") + addURL(u, isLic) + i = end + 1 + } + _ = licNeedles + return licURLs, mediaURLs, hosts +} + +func parseURLHost(raw string) (string, error) { + if !strings.Contains(raw, "://") { + raw = "https://" + raw + } + // tiny parse without importing net/url cycle issues — use net/url + u, err := httpParseHost(raw) + return u, err +} + +func httpParseHost(raw string) (string, error) { + // local helper using strings to avoid extra deps in hot path + rest := raw + if i := strings.Index(rest, "://"); i >= 0 { + rest = rest[i+3:] + } + if i := strings.IndexAny(rest, "/?#"); i >= 0 { + rest = rest[:i] + } + rest = stripHostPort(rest) + if rest == "" { + return "", fmt.Errorf("no host") + } + return rest, nil +} + +func hostIsLoopback(host string) bool { + host = strings.TrimSpace(strings.Trim(host, "[]")) + if host == "" { + return false + } + if host == "localhost" || host == "127.0.0.1" || host == "::1" || host == "0.0.0.0" { + return true + } + if ip := net.ParseIP(host); ip != nil { + return ip.IsLoopback() || ip.IsUnspecified() + } + return false +} + +func stripHostPort(hostport string) string { + hostport = strings.TrimSpace(hostport) + if h, _, err := net.SplitHostPort(hostport); err == nil { + return h + } + return strings.Trim(hostport, "[]") +} + +func requestHost(req *http.Request) string { + if req == nil { + return "" + } + if req.Host != "" { + return req.Host + } + if req.URL != nil && req.URL.Host != "" { + return req.URL.Host + } + return "" +} + +func connectHostFromCtx(ctx *goproxy.ProxyCtx) string { + if ctx == nil { + return "" + } + if m, ok := ctx.UserData.(map[string]string); ok { + return m["connect_host"] + } + if s, ok := ctx.UserData.(string); ok { + return s + } + return "" +} + +// fixLoopbackHost rewrites req Host/URL when they collapsed to the proxy +// loopback address, using the original CONNECT target instead. +func fixLoopbackHost(req *http.Request, connectHost string) { + if req == nil || connectHost == "" { + return + } + realHost := stripHostPort(connectHost) + if realHost == "" || hostIsLoopback(realHost) { + return + } + port := "443" + if _, p, err := net.SplitHostPort(connectHost); err == nil && p != "" { + port = p + } + need := false + if hostIsLoopback(stripHostPort(req.Host)) || req.Host == "" { + need = true + } + if req.URL != nil && (hostIsLoopback(stripHostPort(req.URL.Host)) || req.URL.Host == "") { + need = true + } + if !need { + return + } + log.Printf("[FIX] loopback Host %q → %s", req.Host, realHost) + req.Host = realHost + if req.URL != nil { + req.URL.Host = net.JoinHostPort(realHost, port) + if req.URL.Scheme == "" { + req.URL.Scheme = "https" + } + } +} + +func defaultOutPath() string { + // Prefer /data/local/tmp — always writable by the shell user that runs the proxy. + // /sdcard/Download is often blocked by scoped storage for the shell uid. + for _, p := range []string{ + "/data/local/tmp/appproxy_cap.json", + // Legacy names, still read by hosts provisioned before the rename. + "/data/local/tmp/rte_cap.json", + "/sdcard/Download/rte_cap.json", + "/storage/emulated/0/Download/rte_cap.json", + } { + dir := filepath.Dir(p) + if st, err := os.Stat(dir); err == nil && st.IsDir() { + return p + } + } + return "appproxy_cap.json" +} + +func defaultTrafficPath() string { + return "/data/local/tmp/appproxy_traffic.jsonl" +} + +// formatNeedles mark DRM/manifest traffic for any provider. +var formatNeedles = []string{ + "widevine", + ".mpd", + ".isml", + ".m3u8", + "manifest", + "license", + "licence", + "drm", + "cenc", + "getlicense", + "acquirelicense", +} + +func interesting(host, url string) bool { + h := strings.ToLower(host + " " + url) + for _, needle := range formatNeedles { + if strings.Contains(h, needle) { + return true + } + } + for _, needle := range matchNeedles { + if needle != "" && strings.Contains(h, strings.ToLower(needle)) { + return true + } + } + for _, needle := range scoreHosts { + if needle != "" && strings.Contains(h, strings.ToLower(needle)) { + return true + } + } + return false +} + +// hostIsManifestOrigin reports whether a host was named with --score-host. +func hostIsManifestOrigin(host string) bool { + h := strings.ToLower(host) + for _, needle := range scoreHosts { + if needle != "" && strings.Contains(h, strings.ToLower(needle)) { + return true + } + } + return false +} + +func looksLikeMPD(url string) bool { + u := strings.ToLower(url) + path := u + if i := strings.Index(path, "?"); i >= 0 { + path = path[:i] + } + // Match on path only — metrics trackers embed ".m3u8" in query strings. + return strings.HasSuffix(path, ".mpd") || + strings.Contains(path, "manifest.mpd") || + strings.HasSuffix(path, ".m3u8") || + strings.Contains(path, "playlist-hls") +} + +func looksLikeLicense(method, host, url string) bool { + if !strings.EqualFold(method, http.MethodPost) { + return false + } + h := strings.ToLower(host + " " + url) + for _, needle := range []string{ + "license", "licence", "widevine", "/wv/", "getlicense", + "acquirelicense", "drm", "cenc", "ceneckey", "modular drm", "modulardrm", + } { + if strings.Contains(h, needle) { + return true + } + } + return false +} + +func classifyURL(method, host, url string) string { + if looksLikeMPD(url) { + return "mpd" + } + u := strings.ToLower(url) + if strings.Contains(u, ".isml") || strings.Contains(u, "manifest") || strings.Contains(u, ".m3u8") { + return "manifest" + } + if looksLikeLicense(method, host, url) || strings.Contains(strings.ToLower(host), "widevine") { + return "license" + } + return "other" +} + +type trafficLog struct { + mu sync.Mutex + path string + enabled bool +} + +func (t *trafficLog) write(rec map[string]any) { + if t == nil || !t.enabled || t.path == "" { + return + } + t.mu.Lock() + defer t.mu.Unlock() + raw, err := json.Marshal(rec) + if err != nil { + return + } + f, err := os.OpenFile(t.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + log.Printf("traffic log: %v", err) + return + } + _, _ = f.Write(append(raw, '\n')) + _ = f.Close() +} + +func preferMPD(current, next string) string { + if next == "" { + return current + } + // Reject EPG/schedule/analytics URLs (score <= 0). + if manifestScore(next) <= 0 { + return current + } + if current == "" { + return next + } + if manifestScore(next) >= manifestScore(current) { + return next + } + return current +} + +func manifestScore(u string) int { + lu := strings.ToLower(u) + path := lu + if i := strings.Index(path, "?"); i >= 0 { + path = path[:i] + } + for _, bad := range denyNeedles() { + if strings.Contains(lu, bad) { + return -100 + } + } + s := 0 + if hostIsManifestOrigin(lu) { + s += 50 + } + if strings.Contains(path, "playlist-hls") || strings.Contains(path, "playlist.m3u8") { + s += 40 + } + if strings.Contains(path, "chunklist") { + s += 10 + } + if strings.HasSuffix(path, ".m3u8") { + s += 5 + } + if strings.HasSuffix(path, ".mpd") || strings.Contains(path, "manifest.mpd") { + s += 30 + } + if strings.Contains(path, ".isml") || strings.Contains(path, "/manifest") { + s += 20 + } + return s +} + +func defaultCADir() string { + if exe, err := os.Executable(); err == nil { + return filepath.Dir(exe) + } + return "." +} + +type captureStore struct { + mu sync.Mutex + path string +} + +func (s *captureStore) set(mut func(*capture)) { + s.mu.Lock() + defer s.mu.Unlock() + + c := capture{} + if raw, err := os.ReadFile(s.path); err == nil { + _ = json.Unmarshal(raw, &c) + } + mut(&c) + c.UpdatedAt = time.Now().Format(time.RFC3339) + raw, err := json.MarshalIndent(c, "", " ") + if err != nil { + log.Printf("marshal: %v", err) + return + } + tmp := s.path + ".tmp" + if err := os.WriteFile(tmp, append(raw, '\n'), 0o644); err != nil { + // fallback: write direct + if err2 := os.WriteFile(s.path, append(raw, '\n'), 0o644); err2 != nil { + log.Printf("write %s: %v", s.path, err2) + } + return + } + if err := os.Rename(tmp, s.path); err != nil { + _ = os.WriteFile(s.path, append(raw, '\n'), 0o644) + } +} + +func trim(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} + +// Prefer the short Widevine PSSH (AAAASnBzc2g…) over the giant PlayReady blob. +func firstShortPSSH(mpd string) string { + const marker = "AAAASnBzc2g" + idx := strings.Index(mpd, marker) + if idx < 0 { + // any cenc:pssh content + const open = "") + k := strings.Index(mpd[i:], "") + if j < 0 || k < 0 || k <= j { + return "" + } + return strings.TrimSpace(mpd[i+j+1 : i+k]) + } + end := idx + for end < len(mpd) { + c := mpd[end] + if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '+' || c == '/' || c == '=' { + end++ + continue + } + break + } + return mpd[idx:end] +} + +func loadOrCreateCA(certPath, keyPath string) (*tls.Certificate, error) { + if _, err := os.Stat(certPath); err == nil { + if _, err := os.Stat(keyPath); err == nil { + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, err + } + if len(cert.Certificate) > 0 { + leaf, err := x509.ParseCertificate(cert.Certificate[0]) + if err == nil { + cert.Leaf = leaf + } + } + return &cert, nil + } + } + + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, err + } + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + Organization: []string{caOrganization}, + CommonName: caCommonName, + }, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(10 * 365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLen: 1, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return nil, err + } + certOut, err := os.Create(certPath) + if err != nil { + return nil, err + } + if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: der}); err != nil { + certOut.Close() + return nil, err + } + certOut.Close() + + keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return nil, err + } + if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}); err != nil { + keyOut.Close() + return nil, err + } + keyOut.Close() + + leaf, err := x509.ParseCertificate(der) + if err != nil { + return nil, err + } + return &tls.Certificate{ + Certificate: [][]byte{der}, + PrivateKey: key, + Leaf: leaf, + }, nil +} diff --git a/apps/proxy/device/needles.go b/apps/proxy/device/needles.go new file mode 100644 index 0000000..913f962 --- /dev/null +++ b/apps/proxy/device/needles.go @@ -0,0 +1,56 @@ +package main + +import "strings" + +// CA identity and file names. The legacy names are still honoured when already +// present on a device, so an existing phone does not need a CA reinject. +const ( + caCertName = "appproxy-ca.crt" + caKeyName = "appproxy-ca.key" + legacyCACertName = "rteproxy-ca.crt" + legacyCAKeyName = "rteproxy-ca.key" + caOrganization = "appproxy MITM CA" + caCommonName = "appproxy" +) + +// stringList is a repeatable string flag. +type stringList []string + +func (s *stringList) String() string { return strings.Join(*s, ",") } + +func (s *stringList) Set(v string) error { + for _, part := range strings.Split(v, ",") { + part = strings.TrimSpace(part) + if part != "" { + *s = append(*s, part) + } + } + return nil +} + +var ( + // matchNeedles (--match) mark extra provider traffic as interesting. + matchNeedles stringList + // scoreHosts (--score-host) are this provider's manifest origins. + scoreHosts stringList + // scoreDeny (--score-deny) are provider URLs that are never a manifest. + scoreDeny stringList + // mitmHosts (--mitm-host) force MITM even when a passthrough rule matches. + mitmHosts stringList +) + +// formatDeny are catalog/analytics URL shapes that are never a manifest for any +// provider. Provider-specific noise arrives via --score-deny. +var formatDeny = []string{ + "schedules", "bylistingtime", "maxlistings", "bycallsign", + "/feed.", "playback_config", "config.json", +} + +func denyNeedles() []string { + out := make([]string, 0, len(formatDeny)+len(scoreDeny)) + out = append(out, formatDeny...) + for _, d := range scoreDeny { + out = append(out, strings.ToLower(d)) + } + return out +} diff --git a/apps/proxy/device/passthrough.go b/apps/proxy/device/passthrough.go new file mode 100644 index 0000000..c7d7fbb --- /dev/null +++ b/apps/proxy/device/passthrough.go @@ -0,0 +1,101 @@ +package main + +import ( + "io" + "log" + "net" + "strconv" + "strings" + "sync" +) + +// passthroughAll, when true, splices every host without MITM. Used to prove +// transparent redirect + VPN egress work before narrowing MITM targets. +var passthroughAll bool + +// passthroughDefault, when true (transparent mode), MITM only forceMITM / open.live. +// Avoids breaking connectivity checks on Google/Firebase when Magisk CA is not +// in the app mount namespace. +var passthroughDefault bool + +// forceMITM hosts (lowercase) always MITM even if a passthrough rule matches. +// Use for known license endpoints once identified: -mitm-host license.example.com +var forceMITM []string + +// Hosts that must NOT be MITM'd for playback to work (CDN / media / BBC APIs). +// Transparent mode defaults to passthrough; carve in with open.live / -mitm-host. +func shouldPassthrough(host string) bool { + if passthroughAll { + return true + } + h := strings.ToLower(stripHostPort(host)) + if h == "" { + return false + } + for _, m := range forceMITM { + if h == m || strings.HasSuffix(h, "."+m) { + return false + } + } + // No SNI → only have a destination IP; MITM cert/SNI would be wrong. + if ip := net.ParseIP(h); ip != nil { + return true + } + // MITM open.live (mediaselector) — stream + Widevine licence URLs live here. + // Upstream MUST dial SO_ORIGINAL_DST (VPN fake-IP) or BBC returns geolocation 403. + if h == "open.live.bbc.co.uk" { + return false + } + // Other BBC APIs/CDNs: MITM breaks play; passthrough. + if strings.Contains(h, "bbc.co.uk") || strings.Contains(h, "bbci.co.uk") || + strings.Contains(h, "bbc.com") || strings.Contains(h, "bbci.com") { + return true + } + needles := []string{ + "akamai", "akamaized", "cloudfront.net", "fastly", + "edgesuite", "cmaf", "2cnt.net", "springstreams", + "fingerprint", "optimizely", "appsflyer", "urbanairship", + "googleusercontent", "gvt1.com", "googleapis.com", + "firebaselogging", "crashlytics", "app-measurement", + } + for _, n := range needles { + if strings.Contains(h, n) { + return true + } + } + if passthroughDefault { + return true + } + return false +} + +func handlePassthrough(client net.Conn, host string, port int, dial func(network, addr string) (net.Conn, error)) { + defer client.Close() + target := net.JoinHostPort(host, strconv.Itoa(port)) + up, err := dial("tcp", target) + if err != nil { + log.Printf("[PASS] dial %s: %v", target, err) + return + } + defer up.Close() + log.Printf("[PASS] %s (no MITM)", target) + errc := make(chan error, 2) + var once sync.Once + closeWrite := func(c net.Conn) { + once.Do(func() {}) + if tc, ok := c.(*net.TCPConn); ok { + _ = tc.CloseWrite() + } + } + go func() { + _, err := io.Copy(up, client) + errc <- err + closeWrite(up) + }() + go func() { + _, err := io.Copy(client, up) + errc <- err + closeWrite(client) + }() + <-errc +} diff --git a/apps/proxy/device/push.sh b/apps/proxy/device/push.sh new file mode 100644 index 0000000..0d20584 --- /dev/null +++ b/apps/proxy/device/push.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Push rteproxy to the phone and start it in the background. +set -euo pipefail +cd "$(dirname "$0")" + +BIN="${1:-rteproxy-android-arm64}" +REMOTE="${REMOTE:-/data/local/tmp/rteproxy}" +OUT="${OUT:-/data/local/tmp/rte_cap.json}" +LISTEN="${LISTEN:-:8080}" +DNS="${DNS:-1.1.1.1,1.0.0.1,8.8.8.8}" +ADB="${ADB:-adb}" + +if [[ ! -f "$BIN" ]]; then + echo "missing $BIN — run ./build.sh first" >&2 + exit 1 +fi + +"$ADB" get-state >/dev/null +"$ADB" push "$BIN" "$REMOTE" +"$ADB" shell chmod 755 "$REMOTE" + +# Stop a previous instance (ignore if none) +"$ADB" shell "pkill -f /data/local/tmp/rteproxy" >/dev/null 2>&1 || true +sleep 0.5 + +"$ADB" shell "sh -c '$REMOTE -listen $LISTEN -out $OUT -ca-dir /data/local/tmp -dns $DNS >/data/local/tmp/rteproxy.log 2>&1 &'" +sleep 1 + +echo "--- process ---" +"$ADB" shell "ps -A | grep rteproxy || true" +echo "--- log ---" +"$ADB" shell "cat /data/local/tmp/rteproxy.log || true" +echo +echo "CA cert on device: /data/local/tmp/rteproxy-ca.crt" +echo " adb pull /data/local/tmp/rteproxy-ca.crt ." +echo " → Settings → Security → Install a certificate → CA certificate" +echo +echo "Set Wi‑Fi HTTP proxy to 127.0.0.1${LISTEN}" +echo "Play the stream, then:" +echo " adb pull $OUT /tmp/rte_cap.json" +echo " python getrtelive.py" diff --git a/apps/proxy/device/tproxy_iptables.sh b/apps/proxy/device/tproxy_iptables.sh new file mode 100644 index 0000000..52a61d6 --- /dev/null +++ b/apps/proxy/device/tproxy_iptables.sh @@ -0,0 +1,81 @@ +#!/system/bin/sh +# Transparent redirect: package UID TCP/443 → local appproxy (no Wi‑Fi http_proxy). +# Usage: +# tproxy_iptables.sh start [port] +# tproxy_iptables.sh stop +# tproxy_iptables.sh status + +set -eu +CHAIN=APPROXY_TPROXY +ACTION="${1:-}" + +uid_for_package() { + pkg="$1" + # dumpsys package | grep userId= OR stat on data dir + uid=$(dumpsys package "$pkg" 2>/dev/null | grep -m1 -oE 'userId=[0-9]+' | head -1 | cut -d= -f2 || true) + if [ -z "$uid" ]; then + uid=$(stat -c %u "/data/user/0/$pkg" 2>/dev/null || true) + fi + echo "$uid" +} + +stop_rules() { + iptables -t nat -D OUTPUT -j "$CHAIN" 2>/dev/null || true + iptables -t nat -F "$CHAIN" 2>/dev/null || true + iptables -t nat -X "$CHAIN" 2>/dev/null || true + echo "STOPPED" +} + +start_rules() { + pkg="$1" + port="$2" + uid=$(uid_for_package "$pkg") + if [ -z "$uid" ] || [ "$uid" = "0" ]; then + echo "ERR: cannot resolve uid for package $pkg" >&2 + exit 1 + fi + proxy_uid=$(stat -c %u /data/local/tmp/appproxy 2>/dev/null || echo "") + # Prefer the running process uid if available + if pidof appproxy >/dev/null 2>&1; then + proxy_uid=$(stat -c %u /proc/$(pidof appproxy | awk '{print $1}') 2>/dev/null || echo "$proxy_uid") + fi + + stop_rules >/dev/null + iptables -t nat -N "$CHAIN" + # Never redirect the mitm itself (loop). + if [ -n "$proxy_uid" ]; then + iptables -t nat -A "$CHAIN" -m owner --uid-owner "$proxy_uid" -j RETURN + fi + # Skip localhost / link-local. + iptables -t nat -A "$CHAIN" -d 127.0.0.0/8 -j RETURN + iptables -t nat -A "$CHAIN" -d 10.0.0.0/8 -j RETURN 2>/dev/null || true + # Redirect only the target app's HTTPS. + iptables -t nat -A "$CHAIN" -p tcp -m owner --uid-owner "$uid" --dport 443 -j REDIRECT --to-ports "$port" + iptables -t nat -A OUTPUT -j "$CHAIN" + echo "STARTED pkg=$pkg uid=$uid port=$port proxy_uid=${proxy_uid:-unknown}" +} + +status_rules() { + echo "=== $CHAIN ===" + iptables -t nat -L "$CHAIN" -n -v 2>/dev/null || echo "(no chain)" + echo "=== OUTPUT head ===" + iptables -t nat -L OUTPUT -n -v 2>/dev/null | head -20 +} + +case "$ACTION" in + start) + pkg="${2:?package required}" + port="${3:-8080}" + start_rules "$pkg" "$port" + ;; + stop) + stop_rules + ;; + status) + status_rules + ;; + *) + echo "usage: $0 start [port] | stop | status" >&2 + exit 2 + ;; +esac diff --git a/apps/proxy/device/transparent.go b/apps/proxy/device/transparent.go new file mode 100644 index 0000000..8580e7e --- /dev/null +++ b/apps/proxy/device/transparent.go @@ -0,0 +1,382 @@ +package main + +import ( + "bufio" + "encoding/binary" + "errors" + "fmt" + "io" + "log" + "net" + "strconv" + "strings" + "sync" + "time" + "unsafe" + + "golang.org/x/sys/unix" +) + +// dialFunc is the DNS-aware upstream dialer (same as MITM transport DialContext). +type dialFunc func(network, addr string) (net.Conn, error) + +// serveTransparentAccept accepts iptables-REDIRECTED TCP connections, recovers +// the original destination / SNI, then feeds them into the local HTTP MITM via +// a synthetic CONNECT so UK-VPN routing stays intact (no Wi‑Fi http_proxy). +func serveTransparentAccept(ln net.Listener, mitmAddr string, dial dialFunc) { + log.Printf("transparent accept on %s → MITM %s", ln.Addr(), mitmAddr) + for { + c, err := ln.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) { + return + } + log.Printf("transparent accept: %v", err) + continue + } + go handleTransparent(c, mitmAddr, dial) + } +} + +func handleTransparent(client net.Conn, mitmAddr string, dial dialFunc) { + defer client.Close() + _ = client.SetDeadline(time.Now().Add(30 * time.Second)) + + bc := newBufConn(client) + sni, helloErr := peekSNI(bc) + dstHost, dstPort, dstErr := originalDst(client) + + host := strings.TrimSpace(sni) + port := 443 + if dstErr == nil && dstPort > 0 { + port = dstPort + } + if host == "" { + if dstErr != nil { + log.Printf("transparent: no SNI (%v) and no original dst (%v)", helloErr, dstErr) + return + } + host = dstHost + } + target := net.JoinHostPort(host, strconv.Itoa(port)) + + // CDN / media / mediaselector: splice without MITM so playback works. + if shouldPassthrough(host) { + _ = client.SetDeadline(time.Time{}) + // Re-feed ClientHello: peekSNI left bytes in bc; rebuild a reader. + left := bc.buffered() + var clientR net.Conn = client + if len(left) > 0 { + clientR = &prefixConn{Conn: client, prefix: left} + } + upDial := dial + if upDial == nil { + upDial = func(network, addr string) (net.Conn, error) { + return net.DialTimeout(network, addr, 15*time.Second) + } + } + // Prefer the app's original destination IP (same CDN edge / geo) over re-resolve. + passHost := host + if dstErr == nil && net.ParseIP(dstHost) != nil && !hostIsLoopback(dstHost) { + passHost = dstHost + log.Printf("[PASS] %s → %s:%d (orig-dst)", host, dstHost, port) + } + handlePassthrough(clientR, passHost, port, upDial) + return + } + + log.Printf("[TPROXY] %s → CONNECT %s", client.RemoteAddr(), target) + + mitm, err := net.DialTimeout("tcp", mitmAddr, 5*time.Second) + if err != nil { + log.Printf("transparent dial mitm: %v", err) + return + } + defer mitm.Close() + + // Pass SO_ORIGINAL_DST so MITM upstream dials the app's IP (NordVPN fake-IP + // / same CDN edge). Re-resolving via public DNS breaks BBC geolocation. + var b strings.Builder + fmt.Fprintf(&b, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n", target, target) + if dstErr == nil && net.ParseIP(dstHost) != nil && !hostIsLoopback(dstHost) { + fmt.Fprintf(&b, "X-Appproxy-Orig-Dst: %s\r\n", net.JoinHostPort(dstHost, strconv.Itoa(port))) + } + b.WriteString("\r\n") + if _, err := io.WriteString(mitm, b.String()); err != nil { + log.Printf("transparent CONNECT write: %v", err) + return + } + br := bufio.NewReader(mitm) + status, err := br.ReadString('\n') + if err != nil { + log.Printf("transparent CONNECT read: %v", err) + return + } + if !strings.Contains(status, "200") { + rest, _ := io.ReadAll(io.LimitReader(br, 512)) + log.Printf("transparent CONNECT rejected: %s%s", status, rest) + return + } + // Drain remaining response headers. + for { + line, err := br.ReadString('\n') + if err != nil || line == "\r\n" || line == "\n" { + break + } + } + + _ = client.SetDeadline(time.Time{}) + _ = mitm.SetDeadline(time.Time{}) + + // Any buffered ClientHello bytes must go to the MITM first. + var once sync.Once + left := bc.buffered() + if len(left) > 0 { + if _, err := mitm.Write(left); err != nil { + log.Printf("transparent hello write: %v", err) + return + } + } + + errc := make(chan error, 2) + go func() { + _, err := io.Copy(mitm, bc) + errc <- err + once.Do(func() { + if tc, ok := mitm.(*net.TCPConn); ok { + _ = tc.CloseWrite() + } + }) + }() + go func() { + _, err := io.Copy(client, br) + errc <- err + if tc, ok := client.(*net.TCPConn); ok { + _ = tc.CloseWrite() + } + }() + <-errc +} + +// prefixConn emits prefix once, then reads from Conn. +type prefixConn struct { + net.Conn + prefix []byte + i int +} + +func (p *prefixConn) Read(b []byte) (int, error) { + if p.i < len(p.prefix) { + n := copy(b, p.prefix[p.i:]) + p.i += n + return n, nil + } + return p.Conn.Read(b) +} + +type bufConn struct { + net.Conn + r *bufio.Reader +} + +func newBufConn(c net.Conn) *bufConn { + return &bufConn{Conn: c, r: bufio.NewReaderSize(c, 4096)} +} + +func (b *bufConn) Read(p []byte) (int, error) { return b.r.Read(p) } + +func (b *bufConn) buffered() []byte { + n := b.r.Buffered() + if n == 0 { + return nil + } + buf, _ := b.r.Peek(n) + out := make([]byte, len(buf)) + copy(out, buf) + // Consume so later Read does not duplicate. + _, _ = b.r.Discard(n) + return out +} + +// peekSNI reads a TLS ClientHello (via Peek) and returns the SNI hostname. +func peekSNI(bc *bufConn) (string, error) { + hdr, err := bc.r.Peek(5) + if err != nil { + return "", err + } + if hdr[0] != 0x16 { // handshake + return "", fmt.Errorf("not TLS handshake (type=%d)", hdr[0]) + } + recLen := int(hdr[3])<<8 | int(hdr[4]) + need := 5 + recLen + if need > 16*1024 { + return "", fmt.Errorf("ClientHello too large (%d)", need) + } + // Wait until full record is buffered. + deadline := time.Now().Add(5 * time.Second) + for bc.r.Buffered() < need && time.Now().Before(deadline) { + _ = bc.Conn.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) + _, _ = bc.r.Peek(need) + } + _ = bc.Conn.SetReadDeadline(time.Time{}) + if bc.r.Buffered() < need { + need = bc.r.Buffered() + } + data, err := bc.r.Peek(need) + if err != nil { + return "", err + } + return parseClientHelloSNI(data) +} + +func parseClientHelloSNI(rec []byte) (string, error) { + if len(rec) < 5+4 { + return "", fmt.Errorf("short record") + } + if rec[0] != 0x16 || rec[5] != 0x01 { // handshake + client_hello + return "", fmt.Errorf("not ClientHello") + } + // Skip: record hdr(5) + hs type(1) + hs len(3) + client version(2) + random(32) + i := 5 + 1 + 3 + 2 + 32 + if i >= len(rec) { + return "", fmt.Errorf("truncated ClientHello") + } + // session id + sidLen := int(rec[i]) + i += 1 + sidLen + if i+2 > len(rec) { + return "", fmt.Errorf("truncate cipher suites") + } + csLen := int(rec[i])<<8 | int(rec[i+1]) + i += 2 + csLen + if i+1 > len(rec) { + return "", fmt.Errorf("truncate compression") + } + compLen := int(rec[i]) + i += 1 + compLen + if i+2 > len(rec) { + return "", nil // no extensions + } + extLen := int(rec[i])<<8 | int(rec[i+1]) + i += 2 + end := i + extLen + if end > len(rec) { + end = len(rec) + } + for i+4 <= end { + typ := int(rec[i])<<8 | int(rec[i+1]) + l := int(rec[i+2])<<8 | int(rec[i+3]) + i += 4 + if i+l > end { + break + } + if typ == 0 { // server_name + return parseSNIExtension(rec[i : i+l]) + } + i += l + } + return "", nil +} + +func parseSNIExtension(b []byte) (string, error) { + if len(b) < 2 { + return "", nil + } + listLen := int(b[0])<<8 | int(b[1]) + i := 2 + end := 2 + listLen + if end > len(b) { + end = len(b) + } + for i+3 <= end { + nameType := b[i] + nameLen := int(b[i+1])<<8 | int(b[i+2]) + i += 3 + if i+nameLen > end { + break + } + if nameType == 0 { + return string(b[i : i+nameLen]), nil + } + i += nameLen + } + return "", nil +} + +func originalDst(conn net.Conn) (host string, port int, err error) { + tcp, ok := conn.(*net.TCPConn) + if !ok { + return "", 0, fmt.Errorf("not TCP") + } + rc, err := tcp.SyscallConn() + if err != nil { + return "", 0, err + } + var ( + ip net.IP + prt int + err2 error + ) + cerr := rc.Control(func(fd uintptr) { + ip, prt, err2 = getOrigDstFD(int(fd)) + }) + if cerr != nil { + return "", 0, cerr + } + if err2 != nil { + return "", 0, err2 + } + return ip.String(), prt, nil +} + +func getOrigDstFD(fd int) (net.IP, int, error) { + // Try IPv6 structure first (works for IPv4-mapped too on many kernels). + if ip, port, err := getOrigDstIPv6(fd); err == nil { + return ip, port, nil + } + return getOrigDstIPv4(fd) +} + +func getOrigDstIPv4(fd int) (net.IP, int, error) { + const soOriginalDst = 80 + var addr unix.RawSockaddrInet4 + sz := uint32(unsafe.Sizeof(addr)) + _, _, errno := unix.Syscall6( + unix.SYS_GETSOCKOPT, + uintptr(fd), + uintptr(unix.IPPROTO_IP), + uintptr(soOriginalDst), + uintptr(unsafe.Pointer(&addr)), + uintptr(unsafe.Pointer(&sz)), + 0, + ) + if errno != 0 { + return nil, 0, errno + } + ip := net.IPv4(addr.Addr[0], addr.Addr[1], addr.Addr[2], addr.Addr[3]) + port := int(binary.BigEndian.Uint16((*[2]byte)(unsafe.Pointer(&addr.Port))[:])) + return ip, port, nil +} + +func getOrigDstIPv6(fd int) (net.IP, int, error) { + const soOriginalDst = 80 + var addr unix.RawSockaddrInet6 + sz := uint32(unsafe.Sizeof(addr)) + _, _, errno := unix.Syscall6( + unix.SYS_GETSOCKOPT, + uintptr(fd), + uintptr(unix.IPPROTO_IPV6), + uintptr(soOriginalDst), + uintptr(unsafe.Pointer(&addr)), + uintptr(unsafe.Pointer(&sz)), + 0, + ) + if errno != 0 { + return nil, 0, errno + } + ip := make(net.IP, 16) + copy(ip, addr.Addr[:]) + port := int(binary.BigEndian.Uint16((*[2]byte)(unsafe.Pointer(&addr.Port))[:])) + return ip, port, nil +} + diff --git a/apps/proxy/device/utls_dial.go b/apps/proxy/device/utls_dial.go new file mode 100644 index 0000000..f6efb51 --- /dev/null +++ b/apps/proxy/device/utls_dial.go @@ -0,0 +1,77 @@ +package main + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "strings" + "time" + + utls "github.com/refraction-networking/utls" +) + +// dialTLSChrome dials addr with a Chrome-like ClientHello so CDNs (Fastly) +// are less likely to HTTP 403 Go's default TLS fingerprint. +// Handshakes as HTTP/1.1 only so net/http can use the returned conn. +func dialTLSChrome(ctx context.Context, dialCtx func(context.Context, string, string) (net.Conn, error), network, addr string) (net.Conn, error) { + host := serverNameFromAddr(addr) + raw, err := dialCtx(ctx, network, addr) + if err != nil { + return nil, err + } + cfg := &utls.Config{ + ServerName: host, + InsecureSkipVerify: true, + NextProtos: []string{"http/1.1"}, + } + spec, err := utls.UTLSIdToSpec(utls.HelloChrome_120) + if err != nil { + _ = raw.Close() + return dialTLSStd(ctx, dialCtx, network, addr) + } + for i := range spec.Extensions { + if alpn, ok := spec.Extensions[i].(*utls.ALPNExtension); ok { + alpn.AlpnProtocols = []string{"http/1.1"} + } + } + uConn := utls.UClient(raw, cfg, utls.HelloCustom) + if err := uConn.ApplyPreset(&spec); err != nil { + _ = raw.Close() + return nil, fmt.Errorf("utls preset %s: %w", host, err) + } + deadline, ok := ctx.Deadline() + if !ok { + deadline = time.Now().Add(15 * time.Second) + } + _ = raw.SetDeadline(deadline) + if err := uConn.Handshake(); err != nil { + _ = raw.Close() + return nil, fmt.Errorf("utls handshake %s: %w", host, err) + } + _ = raw.SetDeadline(time.Time{}) + return uConn, nil +} + +func dialTLSStd(ctx context.Context, dialCtx func(context.Context, string, string) (net.Conn, error), network, addr string) (net.Conn, error) { + host := serverNameFromAddr(addr) + raw, err := dialCtx(ctx, network, addr) + if err != nil { + return nil, err + } + cfg := &tls.Config{ServerName: host, InsecureSkipVerify: true, NextProtos: []string{"http/1.1"}} + c := tls.Client(raw, cfg) + if err := c.HandshakeContext(ctx); err != nil { + _ = raw.Close() + return nil, err + } + return c, nil +} + +func serverNameFromAddr(addr string) string { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return strings.TrimSpace(addr) + } + return host +} diff --git a/apps/proxy/go.mod b/apps/proxy/go.mod new file mode 100644 index 0000000..859bedf --- /dev/null +++ b/apps/proxy/go.mod @@ -0,0 +1,7 @@ +module drmdecryption/apps/proxy + +go 1.25.0 + +require drmdecryption v0.0.0 + +replace drmdecryption => ../pkg diff --git a/apps/proxy/proxyctlcmd/proxyctlcmd.go b/apps/proxy/proxyctlcmd/proxyctlcmd.go new file mode 100644 index 0000000..64eb0d4 --- /dev/null +++ b/apps/proxy/proxyctlcmd/proxyctlcmd.go @@ -0,0 +1,468 @@ +// proxyctl — host-side CLI for the on-device HTTPS MITM (appproxy). +// +// Build the android binary, push it, install/reinject the CA, start/stop the +// proxy, run discover mode, and pull captures into outputs/discover/. +// Package proxyctlcmd is the on-device MITM host CLI, exposed as a library so the +// single drm binary can host it as a subcommand. +package proxyctlcmd + +import ( + "flag" + "fmt" + "os" + "os/exec" + "os/signal" + "path/filepath" + "runtime" + "strings" + "syscall" + "time" + + "drmdecryption/adb" + "drmdecryption/proxy" + "drmdecryption/repo" +) + +// Run dispatches a proxy subcommand. args[0] is the subcommand name. +func Run(args []string) error { + if len(args) < 1 { + Usage() + return fmt.Errorf("proxy: subcommand required") + } + sub, rest := args[0], args[1:] + switch sub { + case "build": + buildCmd(rest) + case "push": + pushCmd(rest) + case "install-ca": + installCACmd(rest) + case "reinject-ca": + reinjectCACmd(rest) + case "start": + startCmd(rest) + case "stop": + stopCmd(rest) + case "discover": + discoverCmd(rest) + case "pull": + pullCmd(rest) + case "clear-proxy": + clearProxyCmd(rest) + case "transparent", "tproxy": + transparentCmd(rest) + case "help", "-h", "--help": + Usage() + default: + Usage() + return fmt.Errorf("proxy: unknown subcommand %q", sub) + } + return nil +} + +// Usage prints the proxy subcommand help. +func Usage() { + fmt.Fprintf(os.Stderr, `proxyctl — on-device MITM (appproxy) host control + +Usage: + proxyctl build cross-compile linux/arm64 → bin/ + apps/proxy/ + proxyctl push [--serial S] push binary to /data/local/tmp/appproxy + proxyctl install-ca [--serial S] [--ca PATH] [--reinject] + push CA + HASH.0; optional Magisk reinject + proxyctl reinject-ca [--serial S] [--hash HASH] + Magisk conscrypt bind only (CA already on device) + proxyctl start [--serial S] [--bin PATH] [--install-ca] [--reinject] + stop old → push → start → set http_proxy + proxyctl stop [--serial S] kill the on-device proxy + clear http_proxy + proxyctl discover [--serial S] [--out DIR] [--skip-build] [--install-ca] [--reinject] + -log-all session; Ctrl+C → outputs/discover/ + proxyctl pull [--serial S] [--out DIR] + pull traffic/cap/log into outputs/discover/ + proxyctl clear-proxy [--serial S] clear global http_proxy (+ stop mitm) + proxyctl transparent --package PKG [--serial S] [--out DIR] [--reinject] + iptables REDIRECT capture (UK VPN OK; no Wi‑Fi proxy) + writes /data/local/tmp/capture// ; adb-pulled to --out + +Artifacts land under outputs/discover// by default. +Transparent captures pull into outputs/transparent// by default. +`) +} + +func clientFrom(fs *flag.FlagSet, args []string) *adb.Client { + serial := fs.String("serial", "", "adb device serial") + _ = fs.Parse(args) + c := adb.New() + if *serial != "" { + c = c.WithSerial(*serial) + } + return c +} + +func buildCmd(args []string) { + fs := flag.NewFlagSet("build", flag.ExitOnError) + _ = fs.Parse(args) + root := repo.Root() + deviceDir := filepath.Join(root, "apps", "proxy", "device") + outLocal := filepath.Join(root, "apps", "proxy", "proxy-android-arm64") + outBin := filepath.Join(root, "bin", "proxy-android-arm64") + + fmt.Println("[*] Building linux/arm64 appproxy...") + cmd := exec.Command("go", "build", "-ldflags=-s -w", "-o", outLocal, ".") + cmd.Dir = deviceDir + cmd.Env = append(os.Environ(), + "GOOS=linux", + "GOARCH=arm64", + "CGO_ENABLED=0", + ) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Fprintf(os.Stderr, "build failed: %v\n", err) + os.Exit(1) + } + _ = os.MkdirAll(filepath.Join(root, "bin"), 0o755) + data, err := os.ReadFile(outLocal) + if err != nil { + fmt.Fprintf(os.Stderr, "read binary: %v\n", err) + os.Exit(1) + } + if err := os.WriteFile(outBin, data, 0o755); err != nil { + fmt.Fprintf(os.Stderr, "copy to bin/: %v\n", err) + os.Exit(1) + } + // Compat name next to the canonical one. + // Legacy copy so hosts that still look for the old name keep working. + _ = os.WriteFile(filepath.Join(root, "apps", "proxy", "rteproxy-android-arm64"), data, 0o755) + fmt.Println("[+] apps/proxy/proxy-android-arm64") + fmt.Println("[+] bin/proxy-android-arm64") +} + +func pushCmd(args []string) { + fs := flag.NewFlagSet("push", flag.ExitOnError) + bin := fs.String("bin", "", "local proxy binary (default: auto)") + c := clientFrom(fs, args) + if err := c.EnsureDevice(); err != nil { + fatal(err) + } + local := *bin + if local == "" { + local = proxy.FindLocalBin("") + } + if local == "" { + fatal(fmt.Errorf("no proxy binary — run: proxyctl build")) + } + fmt.Printf("[*] Pushing %s → %s\n", local, proxy.RemoteBin) + if err := c.Push(local, proxy.RemoteBin); err != nil { + fatal(err) + } + _, _ = c.Shell("chmod", "755", proxy.RemoteBin) + fmt.Println("[+] pushed") +} + +func installCACmd(args []string) { + fs := flag.NewFlagSet("install-ca", flag.ExitOnError) + ca := fs.String("ca", "", "local CA PEM (default: the CA pulled from the device)") + reinject := fs.Bool("reinject", false, "Magisk-reinject into conscrypt after push") + c := clientFrom(fs, args) + if err := c.EnsureDevice(); err != nil { + fatal(err) + } + hash, err := proxy.InstallCA(c, *ca, *reinject) + if err != nil { + fatal(err) + } + fmt.Printf("[+] CA hash %s installed on device\n", hash) +} + +func reinjectCACmd(args []string) { + fs := flag.NewFlagSet("reinject-ca", flag.ExitOnError) + hash := fs.String("hash", proxy.DefaultCAHash, "Android CA subject_hash_old") + c := clientFrom(fs, args) + if err := c.EnsureDevice(); err != nil { + fatal(err) + } + proxy.ReinjectCA(c, *hash) +} + +func startCmd(args []string) { + fs := flag.NewFlagSet("start", flag.ExitOnError) + bin := fs.String("bin", "", "local proxy binary (default: auto)") + installCA := fs.Bool("install-ca", false, "push CA + HASH.0 before start") + reinject := fs.Bool("reinject", false, "Magisk-reinject CA (implies -install-ca)") + c := clientFrom(fs, args) + if err := c.EnsureDevice(); err != nil { + fatal(err) + } + if *reinject { + *installCA = true + } + if *installCA { + if _, err := proxy.InstallCA(c, "", *reinject); err != nil { + fatal(err) + } + } + local := *bin + if local == "" { + local = proxy.FindLocalBin("") + } + if local == "" { + fatal(fmt.Errorf("no proxy binary — run: proxyctl build")) + } + if err := proxy.PushAndStart(c, local); err != nil { + fatal(err) + } + if err := proxy.EnsureHTTPProxy(c, ""); err != nil { + fatal(err) + } +} + +func stopCmd(args []string) { + fs := flag.NewFlagSet("stop", flag.ExitOnError) + c := clientFrom(fs, args) + proxy.Stop(c) + proxy.ClearHTTPProxy(c) +} + +func clearProxyCmd(args []string) { + fs := flag.NewFlagSet("clear-proxy", flag.ExitOnError) + c := clientFrom(fs, args) + proxy.ClearHTTPProxy(c) +} + +func transparentCmd(args []string) { + fs := flag.NewFlagSet("transparent", flag.ExitOnError) + pkg := fs.String("package", "", "app package to redirect (e.g. bbc.iplayer.android)") + out := fs.String("out", "", "host pull dir (default: outputs/transparent/)") + bin := fs.String("bin", "", "local proxy binary (default: auto)") + reinject := fs.Bool("reinject", false, "Magisk-reinject CA before start (slow; CA usually already mounted)") + port := fs.String("port", "8080", "transparent listen port on device") + noTail := fs.Bool("no-tail", false, "start only; do not wait / pull on Ctrl+C") + c := clientFrom(fs, args) + if err := c.EnsureDevice(); err != nil { + fatal(err) + } + if strings.TrimSpace(*pkg) == "" { + fatal(fmt.Errorf("--package is required (e.g. --package bbc.iplayer.android)")) + } + local := *bin + if local == "" { + local = proxy.FindLocalBin("") + } + if local == "" { + fatal(fmt.Errorf("no proxy binary — run: proxyctl build")) + } + + // Never leave a stale Wi‑Fi proxy when using transparent mode. + proxy.ClearHTTPProxy(c) + + remoteDir := "/data/local/tmp/capture/" + *pkg + fmt.Printf("[*] Transparent capture for %s → %s\n", *pkg, remoteDir) + if err := proxy.StartTransparent(c, local, *pkg, *port, remoteDir, *reinject); err != nil { + fatal(err) + } + fmt.Println("[+] running. Leave UK VPN ON. Open the app and play.") + fmt.Println(" Ctrl+C → stop iptables + pull captures") + if *noTail { + return + } + + dest := *out + if dest == "" { + dest = filepath.Join(repo.Root(), "outputs", "transparent", time.Now().Format("20060102-150405")) + } + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt) + <-sig + fmt.Println("\n[*] Stopping transparent capture...") + proxy.StopTransparent(c) + _ = os.MkdirAll(dest, 0o755) + if err := proxy.PullDir(c, remoteDir, dest); err != nil { + fmt.Fprintf(os.Stderr, "pull: %v\n", err) + } else { + fmt.Println("[+] pulled into", dest) + } +} + +func pullCmd(args []string) { + fs := flag.NewFlagSet("pull", flag.ExitOnError) + out := fs.String("out", "", "destination dir (default: outputs/discover/)") + c := clientFrom(fs, args) + if err := c.EnsureDevice(); err != nil { + fatal(err) + } + dest := *out + if dest == "" { + dest = proxy.DiscoverOutDir("", "") + } + if err := proxy.PullCaptures(c, dest); err != nil { + fatal(err) + } + fmt.Println("[+] pulled into", dest) +} + +func discoverCmd(args []string) { + fs := flag.NewFlagSet("discover", flag.ExitOnError) + out := fs.String("out", "", "destination dir (default: outputs/discover/)") + skipBuild := fs.Bool("skip-build", false, "do not rebuild the android binary") + installCA := fs.Bool("install-ca", true, "push CA + HASH.0 before discover") + reinject := fs.Bool("reinject", true, "Magisk-reinject CA (default on for discover)") + noTail := fs.Bool("no-tail", false, "start only; do not tail / wait for Ctrl+C") + listen := fs.String("listen", ":8080", "proxy listen address on device") + pkgForce := fs.String("force-stop", "", "package to force-stop after CA reinject (so it inherits the new mount)") + c := clientFrom(fs, args) + + root := repo.Root() + dest := *out + if dest == "" { + dest = proxy.DiscoverOutDir(root, "") + } + _ = os.MkdirAll(dest, 0o755) + + if !*skipBuild { + buildCmd(nil) + } + + if err := c.EnsureDevice(); err != nil { + fatal(err) + } + fmt.Println("[*] Device:", c.Out("get-serialno")) + + local := proxy.FindLocalBin(root) + if local == "" { + fatal(fmt.Errorf("no proxy binary — run: proxyctl build")) + } + + if *installCA || *reinject { + if _, err := proxy.InstallCA(c, "", *reinject); err != nil { + fmt.Fprintf(os.Stderr, "[!] install-ca: %v\n", err) + } + } + if *pkgForce != "" { + c.ForceStop(*pkgForce) + fmt.Printf("[*] Force-stopped %s\n", *pkgForce) + } + + proxy.Stop(c) + fmt.Printf("[*] Pushing %s → %s (discover / -log-all)\n", local, proxy.RemoteBin) + if err := c.Push(local, proxy.RemoteBin); err != nil { + fatal(err) + } + _, _ = c.Shell("chmod", "755", proxy.RemoteBin) + _, _ = c.Shell("rm", "-f", proxy.RemoteLog, proxy.RemoteCap, proxy.RemoteTraffic) + + starter := "#!/system/bin/sh\n" + + "exec " + proxy.RemoteBin + + " -listen " + *listen + + " -out " + proxy.RemoteCap + + " -ca-dir /data/local/tmp" + + " -dns 1.1.1.1,1.0.0.1,8.8.8.8,192.168.1.1" + + " -log-all -traffic " + proxy.RemoteTraffic + + " -v >>" + proxy.RemoteLog + " 2>&1\n" + tmp := filepath.Join(os.TempDir(), "start_appproxy_discover.sh") + if err := os.WriteFile(tmp, []byte(starter), 0o755); err != nil { + fatal(err) + } + defer os.Remove(tmp) + if err := c.Push(tmp, "/data/local/tmp/start_appproxy.sh"); err != nil { + fatal(err) + } + _, _ = c.Shell("chmod", "755", "/data/local/tmp/start_appproxy.sh") + _, _, _ = c.Run("shell", "setsid /data/local/tmp/start_appproxy.sh /dev/null 2>&1 &") + + ok := false + var pid, head string + for i := 0; i < 12; i++ { + time.Sleep(400 * time.Millisecond) + pid = c.Out("shell", "pidof", "appproxy") + if pid == "" { + // Legacy process name. + pid = c.Out("shell", "pidof", "rteproxy") + } + head = c.Out("shell", "head", "-20", proxy.RemoteLog) + if pid != "" && containsListening(head) { + ok = true + break + } + } + if head != "" { + fmt.Println(head) + } + if !ok { + fmt.Fprintln(os.Stderr, c.Out("shell", "cat", proxy.RemoteLog)) + fatal(fmt.Errorf("appproxy failed to start")) + } + fmt.Printf("[+] appproxy pid=%s\n", pid) + + if err := proxy.EnsureHTTPProxy(c, ""); err != nil { + fatal(err) + } + + fmt.Println() + fmt.Println("=== Discover mode ready ===") + fmt.Println("1. Unlock the phone and open the target app.") + fmt.Println("2. Start playback so DRM + manifest traffic flows.") + fmt.Println("3. Ctrl+C stops the tail and pulls captures.") + fmt.Println() + fmt.Println("Local folder:", dest) + fmt.Println() + + if *noTail { + fmt.Println("Started without tail (-no-tail). Pull later with: proxyctl pull") + return + } + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM) + + tailDone := make(chan struct{}) + go func() { + defer close(tailDone) + cmd := exec.Command(c.Bin, append(serialArgs(c), "shell", "tail", "-f", proxy.RemoteLog)...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + _ = cmd.Run() + }() + + select { + case <-sig: + fmt.Println() + fmt.Println("[*] Stopping tail...") + case <-tailDone: + } + + fmt.Println("[*] Pulling captures into", dest) + _ = proxy.PullCaptures(c, dest) + proxy.ClearHTTPProxy(c) + fmt.Println("[+] Done. Inspect:") + entries, _ := os.ReadDir(dest) + for _, e := range entries { + info, _ := e.Info() + size := int64(0) + if info != nil { + size = info.Size() + } + fmt.Printf(" %s (%d bytes)\n", e.Name(), size) + } + tip := filepath.Join(dest, "appproxy_traffic.jsonl") + if runtime.GOOS == "windows" { + fmt.Printf("Tip: Select-String -Path '%s' -Pattern 'mpd|license|widevine|manifest'\n", tip) + } else { + fmt.Printf("Tip: grep -E 'mpd|license|widevine|manifest' %s\n", tip) + } +} + +func serialArgs(c *adb.Client) []string { + if c.Serial == "" { + return nil + } + return []string{"-s", c.Serial} +} + +func containsListening(s string) bool { + return strings.Contains(strings.ToLower(s), "listening") +} + +func fatal(err error) { + fmt.Fprintf(os.Stderr, "proxyctl: %v\n", err) + os.Exit(1) +} diff --git a/apps/proxy/rteproxy-ca.crt b/apps/proxy/rteproxy-ca.crt new file mode 100644 index 0000000..3fa9286 --- /dev/null +++ b/apps/proxy/rteproxy-ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSjCCAjKgAwIBAgIQYdaH1QnfE8PmjHy1H12hXDANBgkqhkiG9w0BAQsFADAu +MRkwFwYDVQQKExBydGVwcm94eSBNSVRNIENBMREwDwYDVQQDEwhydGVwcm94eTAe +Fw0yNjEwMDExNTU1MDhaFw0zNjA5MjgxNjU1MDhaMC4xGTAXBgNVBAoTEHJ0ZXBy +b3h5IE1JVE0gQ0ExETAPBgNVBAMTCHJ0ZXByb3h5MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEA2WS523uzgge/trRY6YXvtSa8JfTizzdKF0OLps8Dhnq8 +9mQMVuCTdgiQyxRk75bJ6k0e/3NePEE8V2/GAeYtFtiqAC5p82d42Bt6wuXADgsH ++tzBuuIr04w5KkUlWV6sI7BVBNE4lD8He56xkfRHlZMYb2anbAC51AQEmTR0Mu3u +ep37UUo0xcrDI+oBh+mLWF5uVgt2XnxmIQE2r95dwKqomWVgzGsc1EkZoPQyhcJX +xE0f71Dtb2zM7GcWaHzDjVZTasqtsY6ISy2v6m063GO+QDpW3GGWqlPOxTyawWox +j4NS4DyBFnpRtFTTMJKq731dQHh5nyVqD1hBtLyViQIDAQABo2QwYjAOBgNVHQ8B +Af8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMBIGA1UdEwEB +/wQIMAYBAf8CAQEwHQYDVR0OBBYEFPqzqtGYBgxFzTT4O6roQjvof7xnMA0GCSqG +SIb3DQEBCwUAA4IBAQCuoLG8GRdCIc4HeM6MyEMQU+P2PiMi6K1jJOZo4ArT4bAB +5OlOphvduzRHMsszl+V+XliwfGkK20L50ykcmj9KR3TseMMelJWOqHKzV7H5yyKj +Hsz2jXGHKQIMs4p0thSxsbwcewIoIjwoiFFs5Ji0l3U9kc1CzEtuixgxGovJHTN9 +W9LoU4mjFHUWC08+n7jtAazXvzhOOII37P4y2v6AFhVC0yiNUu407p3AjRP2Eyeq +p8YOTnncsnpdpjv11s4mZRbF4Jpp6jvNhePgdNqMZAcov+c+L1KIWui0VBzF4XQI +AaENO1ApSPIXlM19FCVNPNufMEPAfbIq/eMZyg7J +-----END CERTIFICATE----- diff --git a/apps/streamd/README.md b/apps/streamd/README.md new file mode 100644 index 0000000..5ac1751 --- /dev/null +++ b/apps/streamd/README.md @@ -0,0 +1,65 @@ +# streamd + +Control plane: REST API, SQLite, dashboard and media supervisor. Hosted by the +single binary as `drm serve`. + +```text +apps/streamd/ + servecmd/ the command body, imported by apps/cli + internal/api HTTP handlers + internal/db SQLite store + internal/ui dashboard (index.html, app.js, app.css) + internal/supervisor downloader + ffmpeg supervision + internal/ws agent heartbeat +``` + +## Run + +```bash +./bin/drm serve --bind 127.0.0.1:8083 --data .cache/streamd --token SECRET +``` + +Port 8083 by default, to stay clear of anything already on 8080/8081. Media tools +resolve from flags, then env (`NRE_PATH` / `FFMPEG_PATH` / `MP4DECRYPT_PATH`), then +`bin/`, then `PATH` — no machine-specific paths are baked in. + +Open `http://HOST:8083/` and put the token in the header box for mutating actions. + +## API + +| Method | Path | Notes | +|---|---|---| +| GET | `/api/health` | liveness + uptime | +| GET | `/api/apps` | compiled-in app modules, channels, registered rewriters | +| GET | `/api/streams` | list + runtime (the agent polls this) | +| POST | `/api/streams` | create | +| GET/PATCH/DELETE | `/api/streams/:id` | detail / edit / delete | +| POST | `/api/streams/:id/start\|stop\|restart` | desired state | +| POST | `/api/streams/:id/credentials` | `{mpd,key,pssh?,auth?,pid?}`, restarts if enabled | +| POST | `/api/streams/:id/claim` | agent lease (`agent_id`, TTL ~3m) | +| POST | `/api/streams/:id/claim/release` | release lease | + +Mutating routes need `Authorization: Bearer TOKEN` (or `X-Streamd-Token`, or +`?token=`). HLS output is served from `--data/www` at `/hls//index.m3u8`. + +## No provider knowledge + +The schema has no provider defaults, and the dashboard's app/channel pickers come +from `/api/apps`. Per-stream downloader settings resolve at start time: + +```text +stored row -> the stream's app module (app.StreamDefaults) -> neutral defaults +``` + +The manifest rewriter is a registry lookup on the stream's `rewriter` column, and +HLS is detected from the manifest shape — not from an app or stream name. + +## Status + +| Area | State | +|---|---| +| serve + SQLite + CRUD UI + claims + credentials | done | +| `/api/apps` + module-driven defaults | done | +| Real downloader/ffmpeg workers + playlist health | partial — least exercised path | + +**Full guide: [docs/streamd.md](../../docs/streamd.md)** diff --git a/apps/streamd/go.mod b/apps/streamd/go.mod new file mode 100644 index 0000000..4064060 --- /dev/null +++ b/apps/streamd/go.mod @@ -0,0 +1,23 @@ +module drmdecryption/apps/streamd + +go 1.25.0 + +require ( + drmdecryption v0.0.0 + github.com/gorilla/websocket v1.5.3 + modernc.org/sqlite v1.34.5 +) + +replace drmdecryption => ../pkg + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.42.0 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect +) diff --git a/apps/streamd/go.sum b/apps/streamd/go.sum new file mode 100644 index 0000000..3dff215 --- /dev/null +++ b/apps/streamd/go.sum @@ -0,0 +1,45 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/apps/streamd/internal/api/api.go b/apps/streamd/internal/api/api.go new file mode 100644 index 0000000..f355d28 --- /dev/null +++ b/apps/streamd/internal/api/api.go @@ -0,0 +1,326 @@ +package api + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strconv" + "strings" + "time" + + "drmdecryption/apps/streamd/internal/db" +) + +type Supervisor interface { + Start(id int64) error + Stop(id int64) error + Restart(id int64) error +} + +type Handler struct { + Store *db.Store + Token string + Supervisor Supervisor + StartedAt time.Time +} + +func (h *Handler) Mount(mux *http.ServeMux) { + mux.HandleFunc("/api/health", h.health) + mux.HandleFunc("/api/streams", h.streams) + mux.HandleFunc("/api/streams/", h.streamAction) + mux.HandleFunc("/api/apps", h.apps) +} + +func (h *Handler) auth(w http.ResponseWriter, r *http.Request) bool { + if h.Token == "" { + return true + } + if r.Method == http.MethodGet { + return true + } + auth := r.Header.Get("Authorization") + tok := strings.TrimPrefix(auth, "Bearer ") + if tok == "" { + tok = r.Header.Get("X-Streamd-Token") + } + if tok == "" { + tok = r.URL.Query().Get("token") + } + if tok != h.Token { + writeErr(w, http.StatusUnauthorized, "unauthorized") + return false + } + return true +} + +func (h *Handler) health(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeErr(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "ok": true, + "service": "streamd", + "uptime_s": time.Since(h.StartedAt).Seconds(), + "started_at": h.StartedAt.UTC().Format(time.RFC3339), + }) +} + +func (h *Handler) streams(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + list, err := h.Store.ListStreams() + if err != nil { + writeErr(w, http.StatusInternalServerError, err.Error()) + return + } + if list == nil { + list = []db.Stream{} + } + writeJSON(w, http.StatusOK, map[string]any{"streams": list}) + case http.MethodPost: + if !h.auth(w, r) { + return + } + var in db.CreateStream + if err := readJSON(r, &in); err != nil { + writeErr(w, http.StatusBadRequest, err.Error()) + return + } + st, err := h.Store.CreateStream(in) + if err != nil { + writeErr(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusCreated, st) + default: + writeErr(w, http.StatusMethodNotAllowed, "method not allowed") + } +} + +func (h *Handler) streamAction(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/api/streams/") + path = strings.Trim(path, "/") + if path == "" { + h.streams(w, r) + return + } + parts := strings.Split(path, "/") + id, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + writeErr(w, http.StatusBadRequest, "invalid id") + return + } + action := "" + if len(parts) > 1 { + action = parts[1] + } + sub := "" + if len(parts) > 2 { + sub = parts[2] + } + + switch { + case action == "" && r.Method == http.MethodGet: + st, err := h.Store.GetStream(id) + if err != nil { + writeErr(w, http.StatusNotFound, "not found") + return + } + writeJSON(w, http.StatusOK, st) + + case action == "" && (r.Method == http.MethodPatch || r.Method == http.MethodPut): + if !h.auth(w, r) { + return + } + var in db.PatchStream + if err := readJSON(r, &in); err != nil { + writeErr(w, http.StatusBadRequest, err.Error()) + return + } + before, _ := h.Store.GetStream(id) + st, err := h.Store.PatchStream(id, in) + if err != nil { + writeErr(w, http.StatusBadRequest, err.Error()) + return + } + // Editing mpd/key/headers must tear down the live media worker and + // bring up a fresh instance with the new credentials. + if h.Supervisor != nil && credentialsChanged(before, st, in) { + if st.Enabled { + _ = h.Supervisor.Restart(id) + } else { + _ = h.Supervisor.Stop(id) + } + st, _ = h.Store.GetStream(id) + } + writeJSON(w, http.StatusOK, st) + + case action == "" && r.Method == http.MethodDelete: + if !h.auth(w, r) { + return + } + if h.Supervisor != nil { + _ = h.Supervisor.Stop(id) + } + if err := h.Store.DeleteStream(id); err != nil { + writeErr(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]any{"ok": true}) + + case action == "start" && r.Method == http.MethodPost: + if !h.auth(w, r) { + return + } + st, err := h.Store.SetEnabled(id, true) + if err != nil { + writeErr(w, http.StatusNotFound, err.Error()) + return + } + if h.Supervisor != nil { + if err := h.Supervisor.Start(id); err != nil { + writeErr(w, http.StatusInternalServerError, err.Error()) + return + } + st, _ = h.Store.GetStream(id) + } + writeJSON(w, http.StatusOK, st) + + case action == "stop" && r.Method == http.MethodPost: + if !h.auth(w, r) { + return + } + if h.Supervisor != nil { + _ = h.Supervisor.Stop(id) + } + st, err := h.Store.SetEnabled(id, false) + if err != nil { + writeErr(w, http.StatusNotFound, err.Error()) + return + } + writeJSON(w, http.StatusOK, st) + + case action == "restart" && r.Method == http.MethodPost: + if !h.auth(w, r) { + return + } + st, err := h.Store.SetEnabled(id, true) + if err != nil { + writeErr(w, http.StatusNotFound, err.Error()) + return + } + if h.Supervisor != nil { + if err := h.Supervisor.Restart(id); err != nil { + writeErr(w, http.StatusInternalServerError, err.Error()) + return + } + st, _ = h.Store.GetStream(id) + } + writeJSON(w, http.StatusOK, st) + + case action == "credentials" && r.Method == http.MethodPost: + if !h.auth(w, r) { + return + } + var cred db.Credentials + if err := readJSON(r, &cred); err != nil { + writeErr(w, http.StatusBadRequest, err.Error()) + return + } + st, err := h.Store.SetCredentials(id, cred) + if err != nil { + writeErr(w, http.StatusBadRequest, err.Error()) + return + } + _ = h.Store.ReleaseClaim(id, "") + // Always kill the old media worker; start a fresh one when enabled. + if h.Supervisor != nil { + if st.Enabled { + _ = h.Supervisor.Restart(id) + } else { + _ = h.Supervisor.Stop(id) + } + st, _ = h.Store.GetStream(id) + } + writeJSON(w, http.StatusOK, st) + + case action == "claim" && sub == "" && r.Method == http.MethodPost: + if !h.auth(w, r) { + return + } + var body struct { + AgentID string `json:"agent_id"` + TTLSec int `json:"ttl_sec"` + } + _ = readJSON(r, &body) + ttl := time.Duration(body.TTLSec) * time.Second + st, err := h.Store.Claim(id, body.AgentID, ttl) + if err != nil { + if err.Error() == "claimed" { + writeErr(w, http.StatusConflict, "already claimed") + return + } + writeErr(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, st) + + case action == "claim" && sub == "release" && r.Method == http.MethodPost: + if !h.auth(w, r) { + return + } + var body struct { + AgentID string `json:"agent_id"` + } + _ = readJSON(r, &body) + if err := h.Store.ReleaseClaim(id, body.AgentID); err != nil { + writeErr(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]any{"ok": true}) + + default: + writeErr(w, http.StatusNotFound, "not found") + } +} + +func readJSON(r *http.Request, dst any) error { + defer r.Body.Close() + b, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + return err + } + if len(b) == 0 { + return errors.New("empty body") + } + return json.Unmarshal(b, dst) +} + +func writeJSON(w http.ResponseWriter, code int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + _ = enc.Encode(v) +} + +func writeErr(w http.ResponseWriter, code int, msg string) { + writeJSON(w, code, map[string]any{"error": msg}) +} + +// credentialsChanged is true when the patch touched mpd, key, or headers that +// the live NRE/ffmpeg worker is already using. +func credentialsChanged(before, after db.Stream, in db.PatchStream) bool { + if in.MPD != nil && strings.TrimSpace(before.MPD) != strings.TrimSpace(after.MPD) { + return true + } + if in.Key != nil && strings.TrimSpace(before.Key) != strings.TrimSpace(after.Key) { + return true + } + if in.HeadersJSON != nil && strings.TrimSpace(before.HeadersJSON) != strings.TrimSpace(after.HeadersJSON) { + return true + } + return false +} diff --git a/apps/streamd/internal/api/apps.go b/apps/streamd/internal/api/apps.go new file mode 100644 index 0000000..2f0d3d3 --- /dev/null +++ b/apps/streamd/internal/api/apps.go @@ -0,0 +1,47 @@ +package api + +import ( + "net/http" + + "drmdecryption/app" + "drmdecryption/mpd" +) + +// appInfo describes one compiled-in app module to the dashboard, so the UI never +// has to hardcode a provider or channel name. +type appInfo struct { + Name string `json:"name"` + Channels []channelInfo `json:"channels"` + Rewriter string `json:"rewriter"` + HasConfig bool `json:"has_config"` +} + +type channelInfo struct { + ID string `json:"id"` + Label string `json:"label"` +} + +// apps answers GET /api/apps: which providers this binary was built with. +func (h *Handler) apps(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeErr(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + out := []appInfo{} + for _, a := range app.All() { + info := appInfo{Name: a.Name(), Channels: []channelInfo{}, Rewriter: "none"} + for _, ch := range a.Channels() { + info.Channels = append(info.Channels, channelInfo{ID: ch.ID, Label: ch.Label}) + } + // A module with no local values file has no channels to offer yet. + info.HasConfig = len(info.Channels) > 0 + if sd, ok := a.(app.StreamDefaults); ok { + info.Rewriter = sd.RewriterName() + } + out = append(out, info) + } + writeJSON(w, http.StatusOK, map[string]any{ + "apps": out, + "rewriters": mpd.Names(), + }) +} diff --git a/apps/streamd/internal/db/db.go b/apps/streamd/internal/db/db.go new file mode 100644 index 0000000..90bb690 --- /dev/null +++ b/apps/streamd/internal/db/db.go @@ -0,0 +1,504 @@ +package db + +import ( + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +type Store struct { + DB *sql.DB +} + +type Stream struct { + ID int64 `json:"id"` + Name string `json:"name"` + Title string `json:"title"` + App string `json:"app"` + Channel string `json:"channel"` + MPD string `json:"mpd"` + Key string `json:"key"` + HeadersJSON string `json:"headers_json"` + VideoSelect string `json:"video_select"` + AudioSelect string `json:"audio_select"` + Enabled bool `json:"enabled"` + Rewriter string `json:"rewriter"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + + Health string `json:"health,omitempty"` + PID int64 `json:"pid,omitempty"` + PlayPath string `json:"play_path,omitempty"` + UptimeS float64 `json:"uptime_s,omitempty"` + BitrateMbps float64 `json:"bitrate_mbps,omitempty"` + PlaylistAgeS float64 `json:"playlist_age_s,omitempty"` + LastError string `json:"last_error,omitempty"` + RuntimeAt string `json:"runtime_updated_at,omitempty"` + + ClaimedBy string `json:"claimed_by,omitempty"` + ClaimExp string `json:"claim_expires_at,omitempty"` +} + +type CreateStream struct { + Name string `json:"name"` + Title string `json:"title"` + App string `json:"app"` + Channel string `json:"channel"` + MPD string `json:"mpd"` + Key string `json:"key"` + HeadersJSON string `json:"headers_json"` + VideoSelect string `json:"video_select"` + AudioSelect string `json:"audio_select"` + Rewriter string `json:"rewriter"` + Enabled *bool `json:"enabled"` +} + +type PatchStream struct { + Title *string `json:"title"` + App *string `json:"app"` + Channel *string `json:"channel"` + MPD *string `json:"mpd"` + Key *string `json:"key"` + HeadersJSON *string `json:"headers_json"` + VideoSelect *string `json:"video_select"` + AudioSelect *string `json:"audio_select"` + Rewriter *string `json:"rewriter"` + Enabled *bool `json:"enabled"` +} + +type Credentials struct { + MPD string `json:"mpd"` + Key string `json:"key"` + PSSH string `json:"pssh,omitempty"` + Auth string `json:"auth,omitempty"` + PID string `json:"pid,omitempty"` +} + +func Open(dataDir string) (*Store, error) { + if err := os.MkdirAll(dataDir, 0o755); err != nil { + return nil, err + } + path := filepath.Join(dataDir, "streamd.db") + dsn := fmt.Sprintf("file:%s?_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)", filepath.ToSlash(path)) + sqlDB, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, err + } + sqlDB.SetMaxOpenConns(1) + s := &Store{DB: sqlDB} + if err := s.migrate(); err != nil { + _ = sqlDB.Close() + return nil, err + } + return s, nil +} + +func (s *Store) Close() error { + return s.DB.Close() +} + +func (s *Store) migrate() error { + _, err := s.DB.Exec(` +CREATE TABLE IF NOT EXISTS streams ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + title TEXT NOT NULL DEFAULT '', + app TEXT NOT NULL DEFAULT '', + channel TEXT NOT NULL DEFAULT '', + mpd TEXT NOT NULL DEFAULT '', + key_kid_key TEXT NOT NULL DEFAULT '', + headers_json TEXT NOT NULL DEFAULT '{}', + video_select TEXT NOT NULL DEFAULT '', + audio_select TEXT NOT NULL DEFAULT '', + enabled INTEGER NOT NULL DEFAULT 0, + rewriter TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS stream_runtime ( + stream_id INTEGER PRIMARY KEY REFERENCES streams(id) ON DELETE CASCADE, + health TEXT NOT NULL DEFAULT 'stopped', + pid INTEGER NOT NULL DEFAULT 0, + play_path TEXT NOT NULL DEFAULT '', + uptime_s REAL NOT NULL DEFAULT 0, + bitrate_mbps REAL NOT NULL DEFAULT 0, + playlist_age_s REAL NOT NULL DEFAULT 0, + last_error TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL DEFAULT '' +); + +CREATE TABLE IF NOT EXISTS agent_claims ( + stream_id INTEGER PRIMARY KEY REFERENCES streams(id) ON DELETE CASCADE, + agent_id TEXT NOT NULL, + claimed_at TEXT NOT NULL, + expires_at TEXT NOT NULL +); +`) + return err +} + +func now() string { + return time.Now().UTC().Format(time.RFC3339) +} + +func slug(name string) string { + name = strings.TrimSpace(strings.ToLower(name)) + var b strings.Builder + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + case r == '-' || r == '_' || r == ' ': + b.WriteByte('-') + } + } + out := strings.Trim(b.String(), "-") + for strings.Contains(out, "--") { + out = strings.ReplaceAll(out, "--", "-") + } + return out +} + +func (s *Store) ListStreams() ([]Stream, error) { + rows, err := s.DB.Query(` +SELECT s.id, s.name, s.title, s.app, s.channel, s.mpd, s.key_kid_key, s.headers_json, + s.video_select, s.audio_select, s.enabled, s.rewriter, s.created_at, s.updated_at, + COALESCE(r.health,''), COALESCE(r.pid,0), COALESCE(r.play_path,''), + COALESCE(r.uptime_s,0), COALESCE(r.bitrate_mbps,0), COALESCE(r.playlist_age_s,0), + COALESCE(r.last_error,''), COALESCE(r.updated_at,''), + COALESCE(c.agent_id,''), COALESCE(c.expires_at,'') +FROM streams s +LEFT JOIN stream_runtime r ON r.stream_id = s.id +LEFT JOIN agent_claims c ON c.stream_id = s.id +ORDER BY s.name`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Stream + for rows.Next() { + st, err := scanStream(rows) + if err != nil { + return nil, err + } + out = append(out, st) + } + return out, rows.Err() +} + +func (s *Store) GetStream(id int64) (Stream, error) { + row := s.DB.QueryRow(` +SELECT s.id, s.name, s.title, s.app, s.channel, s.mpd, s.key_kid_key, s.headers_json, + s.video_select, s.audio_select, s.enabled, s.rewriter, s.created_at, s.updated_at, + COALESCE(r.health,''), COALESCE(r.pid,0), COALESCE(r.play_path,''), + COALESCE(r.uptime_s,0), COALESCE(r.bitrate_mbps,0), COALESCE(r.playlist_age_s,0), + COALESCE(r.last_error,''), COALESCE(r.updated_at,''), + COALESCE(c.agent_id,''), COALESCE(c.expires_at,'') +FROM streams s +LEFT JOIN stream_runtime r ON r.stream_id = s.id +LEFT JOIN agent_claims c ON c.stream_id = s.id +WHERE s.id = ?`, id) + return scanStream(row) +} + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanStream(row rowScanner) (Stream, error) { + var st Stream + var enabled int + err := row.Scan( + &st.ID, &st.Name, &st.Title, &st.App, &st.Channel, &st.MPD, &st.Key, &st.HeadersJSON, + &st.VideoSelect, &st.AudioSelect, &enabled, &st.Rewriter, &st.CreatedAt, &st.UpdatedAt, + &st.Health, &st.PID, &st.PlayPath, &st.UptimeS, &st.BitrateMbps, &st.PlaylistAgeS, + &st.LastError, &st.RuntimeAt, &st.ClaimedBy, &st.ClaimExp, + ) + if err != nil { + return st, err + } + st.Enabled = enabled != 0 + if st.Health == "" { + st.Health = "stopped" + } + if st.PlayPath == "" { + st.PlayPath = "/hls/" + st.Name + "/index.m3u8" + } + // Hide expired claims in API responses (ExpireClaims cleans them shortly after). + if st.ClaimExp != "" { + if exp, e := time.Parse(time.RFC3339, st.ClaimExp); e == nil && !exp.After(time.Now().UTC()) { + st.ClaimedBy = "" + st.ClaimExp = "" + } + } + return st, nil +} + +func (s *Store) CreateStream(in CreateStream) (Stream, error) { + name := slug(in.Name) + if name == "" { + return Stream{}, fmt.Errorf("name required") + } + if in.HeadersJSON == "" { + in.HeadersJSON = "{}" + } + if !json.Valid([]byte(in.HeadersJSON)) { + return Stream{}, fmt.Errorf("headers_json must be valid JSON") + } + // video_select / audio_select / rewriter are left empty on purpose: the + // supervisor resolves them from the stream's app module at start time, so no + // provider's preferences are baked into the schema. + if in.Title == "" { + in.Title = name + } + enabled := 0 + if in.Enabled != nil && *in.Enabled { + enabled = 1 + } + ts := now() + res, err := s.DB.Exec(` +INSERT INTO streams(name,title,app,channel,mpd,key_kid_key,headers_json,video_select,audio_select,enabled,rewriter,created_at,updated_at) +VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`, + name, in.Title, in.App, in.Channel, in.MPD, in.Key, in.HeadersJSON, + in.VideoSelect, in.AudioSelect, enabled, in.Rewriter, ts, ts) + if err != nil { + return Stream{}, err + } + id, _ := res.LastInsertId() + _, _ = s.DB.Exec(`INSERT OR IGNORE INTO stream_runtime(stream_id,health,play_path,updated_at) VALUES(?,?,?,?)`, + id, "stopped", "/hls/"+name+"/index.m3u8", ts) + return s.GetStream(id) +} + +func (s *Store) PatchStream(id int64, in PatchStream) (Stream, error) { + cur, err := s.GetStream(id) + if err != nil { + return Stream{}, err + } + if in.Title != nil { + cur.Title = *in.Title + } + if in.App != nil { + cur.App = *in.App + } + if in.Channel != nil { + cur.Channel = *in.Channel + } + if in.MPD != nil { + cur.MPD = *in.MPD + } + if in.Key != nil { + cur.Key = *in.Key + } + if in.HeadersJSON != nil { + if !json.Valid([]byte(*in.HeadersJSON)) { + return Stream{}, fmt.Errorf("headers_json must be valid JSON") + } + cur.HeadersJSON = *in.HeadersJSON + } + if in.VideoSelect != nil { + cur.VideoSelect = *in.VideoSelect + } + if in.AudioSelect != nil { + cur.AudioSelect = *in.AudioSelect + } + if in.Rewriter != nil { + cur.Rewriter = *in.Rewriter + } + enabled := 0 + if cur.Enabled { + enabled = 1 + } + if in.Enabled != nil { + if *in.Enabled { + enabled = 1 + } else { + enabled = 0 + } + } + ts := now() + _, err = s.DB.Exec(` +UPDATE streams SET title=?, app=?, channel=?, mpd=?, key_kid_key=?, headers_json=?, + video_select=?, audio_select=?, enabled=?, rewriter=?, updated_at=? WHERE id=?`, + cur.Title, cur.App, cur.Channel, cur.MPD, cur.Key, cur.HeadersJSON, + cur.VideoSelect, cur.AudioSelect, enabled, cur.Rewriter, ts, id) + if err != nil { + return Stream{}, err + } + return s.GetStream(id) +} + +func (s *Store) DeleteStream(id int64) error { + _, err := s.DB.Exec(`DELETE FROM streams WHERE id=?`, id) + return err +} + +func (s *Store) SetEnabled(id int64, enabled bool) (Stream, error) { + v := 0 + if enabled { + v = 1 + } + _, err := s.DB.Exec(`UPDATE streams SET enabled=?, updated_at=? WHERE id=?`, v, now(), id) + if err != nil { + return Stream{}, err + } + health := "stopped" + if enabled { + health = "starting" + } + _, _ = s.DB.Exec(` +INSERT INTO stream_runtime(stream_id,health,play_path,updated_at) +VALUES(?,?, (SELECT '/hls/'||name||'/index.m3u8' FROM streams WHERE id=?), ?) +ON CONFLICT(stream_id) DO UPDATE SET health=excluded.health, play_path=excluded.play_path, updated_at=excluded.updated_at`, + id, health, id, now()) + return s.GetStream(id) +} + +func (s *Store) SetCredentials(id int64, cred Credentials) (Stream, error) { + if strings.TrimSpace(cred.MPD) == "" || strings.TrimSpace(cred.Key) == "" { + return Stream{}, fmt.Errorf("mpd and key required") + } + _, err := s.DB.Exec(`UPDATE streams SET mpd=?, key_kid_key=?, updated_at=? WHERE id=?`, + strings.TrimSpace(cred.MPD), strings.TrimSpace(cred.Key), now(), id) + if err != nil { + return Stream{}, err + } + return s.GetStream(id) +} + +func (s *Store) Claim(id int64, agentID string, ttl time.Duration) (Stream, error) { + if agentID == "" { + agentID = "agent" + } + if ttl <= 0 { + ttl = 90 * time.Second + } + // Drop expired rows first so a dead agent cannot block forever. + _, _ = s.ExpireClaims() + st, err := s.GetStream(id) + if err != nil { + return Stream{}, err + } + nowT := time.Now().UTC() + if st.ClaimedBy != "" && st.ClaimExp != "" { + if exp, e := time.Parse(time.RFC3339, st.ClaimExp); e == nil && exp.After(nowT) && st.ClaimedBy != agentID { + return Stream{}, fmt.Errorf("claimed") + } + } + claimedAt := nowT.Format(time.RFC3339) + expires := nowT.Add(ttl).Format(time.RFC3339) + _, err = s.DB.Exec(` +INSERT INTO agent_claims(stream_id,agent_id,claimed_at,expires_at) VALUES(?,?,?,?) +ON CONFLICT(stream_id) DO UPDATE SET agent_id=excluded.agent_id, claimed_at=excluded.claimed_at, expires_at=excluded.expires_at`, + id, agentID, claimedAt, expires) + if err != nil { + return Stream{}, err + } + return s.GetStream(id) +} + +func (s *Store) ReleaseClaim(id int64, agentID string) error { + if agentID == "" { + _, err := s.DB.Exec(`DELETE FROM agent_claims WHERE stream_id=?`, id) + return err + } + _, err := s.DB.Exec(`DELETE FROM agent_claims WHERE stream_id=? AND agent_id=?`, id, agentID) + return err +} + +// ExpireClaims deletes claims whose expires_at is in the past. Returns rows removed. +func (s *Store) ExpireClaims() (int64, error) { + res, err := s.DB.Exec(`DELETE FROM agent_claims WHERE expires_at <> '' AND expires_at < ?`, now()) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + +// RenewAgentClaims extends expires_at for every claim held by agentID. +func (s *Store) RenewAgentClaims(agentID string, ttl time.Duration) (int64, error) { + if agentID == "" { + return 0, nil + } + if ttl <= 0 { + ttl = 90 * time.Second + } + exp := time.Now().UTC().Add(ttl).Format(time.RFC3339) + res, err := s.DB.Exec(`UPDATE agent_claims SET expires_at=? WHERE agent_id=?`, exp, agentID) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + +// ReleaseAgentClaims drops every claim for agentID (used on WS disconnect). +func (s *Store) ReleaseAgentClaims(agentID string) (int64, error) { + if agentID == "" { + return 0, nil + } + res, err := s.DB.Exec(`DELETE FROM agent_claims WHERE agent_id=?`, agentID) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + +// RuntimePatch updates live worker fields for a stream. +type RuntimePatch struct { + Health string + PID int64 + PlayPath string + UptimeS float64 + BitrateMbps float64 + PlaylistAgeS float64 + LastError string +} + +func (s *Store) SetRuntime(id int64, p RuntimePatch) error { + play := p.PlayPath + if play == "" { + st, err := s.GetStream(id) + if err == nil { + play = "/hls/" + st.Name + "/index.m3u8" + } + } + _, err := s.DB.Exec(` +INSERT INTO stream_runtime(stream_id,health,pid,play_path,uptime_s,bitrate_mbps,playlist_age_s,last_error,updated_at) +VALUES(?,?,?,?,?,?,?,?,?) +ON CONFLICT(stream_id) DO UPDATE SET + health=excluded.health, + pid=excluded.pid, + play_path=excluded.play_path, + uptime_s=excluded.uptime_s, + bitrate_mbps=excluded.bitrate_mbps, + playlist_age_s=excluded.playlist_age_s, + last_error=excluded.last_error, + updated_at=excluded.updated_at`, + id, p.Health, p.PID, play, p.UptimeS, p.BitrateMbps, p.PlaylistAgeS, p.LastError, now()) + return err +} + +// NeedsCapture reports whether an enabled stream looks down / missing creds. +func (st Stream) NeedsCapture() bool { + if !st.Enabled { + return false + } + if strings.TrimSpace(st.MPD) == "" || strings.TrimSpace(st.Key) == "" { + return true + } + switch st.Health { + case "down", "stopped", "starting", "": + return true + } + if st.PlaylistAgeS > 30 { + return true + } + return false +} diff --git a/apps/streamd/internal/supervisor/appdefaults.go b/apps/streamd/internal/supervisor/appdefaults.go new file mode 100644 index 0000000..1a78aba --- /dev/null +++ b/apps/streamd/internal/supervisor/appdefaults.go @@ -0,0 +1,87 @@ +package supervisor + +import ( + "strings" + + "drmdecryption/app" + "drmdecryption/apps/streamd/internal/db" +) + +// streamSettings are the downloader knobs for one stream. They come from the +// stream's app module when it declares them, so the supervisor never has to +// guess from an app name or a stream name. +type streamSettings struct { + VideoSelect string + AudioSelect string + Rewriter string + LiveWait int + TSReadyBytes int64 + // IsHLS drives packaging choices that depend on the manifest format. + IsHLS bool +} + +const ( + defaultLiveWait = 2 + defaultTSReadyBytes = 256 * 1024 + // hlsTSReadyBytes buffers more before probing: an HLS pipe-mux often starts + // with incomplete audio config. + hlsTSReadyBytes = 2 * 1024 * 1024 + hlsLiveWait = 6 +) + +// settingsFor resolves a stream's downloader settings: the app module's +// declarations first, then the stream row, then neutral defaults. +func settingsFor(st db.Stream) streamSettings { + s := streamSettings{ + VideoSelect: strings.TrimSpace(st.VideoSelect), + AudioSelect: strings.TrimSpace(st.AudioSelect), + Rewriter: strings.TrimSpace(st.Rewriter), + LiveWait: defaultLiveWait, + TSReadyBytes: defaultTSReadyBytes, + IsHLS: isHLS(st), + } + if a, err := app.Get(st.App); err == nil { + if sd, ok := a.(app.StreamDefaults); ok { + if s.VideoSelect == "" { + s.VideoSelect = sd.VideoSelect() + } + if s.AudioSelect == "" { + s.AudioSelect = sd.AudioSelect() + } + if s.Rewriter == "" { + s.Rewriter = sd.RewriterName() + } + if n := sd.LiveWaitSeconds(); n > 0 { + s.LiveWait = n + } + if n := sd.TSReadyBytes(); n > 0 { + s.TSReadyBytes = n + } + } + } + if s.IsHLS { + if s.LiveWait == defaultLiveWait { + s.LiveWait = hlsLiveWait + } + if s.TSReadyBytes == defaultTSReadyBytes { + s.TSReadyBytes = hlsTSReadyBytes + } + } + if s.VideoSelect == "" { + s.VideoSelect = "for=best" + } + if s.AudioSelect == "" { + s.AudioSelect = "for=best" + } + if s.Rewriter == "" { + s.Rewriter = "none" + } + return s +} + +// isHLS reports whether a stream's manifest is an HLS playlist, by URL shape. +// No provider host is consulted: a module declares its own needs through +// app.StreamDefaults. +func isHLS(st db.Stream) bool { + return strings.Contains(strings.ToLower(strings.TrimSpace(st.MPD)), ".m3u8") +} diff --git a/apps/streamd/internal/supervisor/appdefaults_test.go b/apps/streamd/internal/supervisor/appdefaults_test.go new file mode 100644 index 0000000..eda8448 --- /dev/null +++ b/apps/streamd/internal/supervisor/appdefaults_test.go @@ -0,0 +1,129 @@ +package supervisor + +import ( + "testing" + + "drmdecryption/adb" + "drmdecryption/app" + "drmdecryption/apps/streamd/internal/db" + "drmdecryption/capture" + "drmdecryption/mpd" +) + +// fakeApp stands in for a compiled-in app module: the supervisor must take its +// declared settings without knowing anything about the provider. +type fakeApp struct { + name string + rewriter string + liveWait int + tsBytes int64 +} + +func (f *fakeApp) Name() string { return f.name } +func (f *fakeApp) Package() string { return "com.example." + f.name } +func (f *fakeApp) LicenseURL() string { return "" } +func (f *fakeApp) Launch(*adb.Client) error { return nil } +func (f *fakeApp) AutoPlay(*adb.Client, string) error { return nil } +func (f *fakeApp) CaptureHints() capture.Hints { return capture.DefaultHints() } +func (f *fakeApp) MPDRewriter() mpd.Rewriter { return mpd.Passthrough{} } +func (f *fakeApp) Channels() []app.Channel { return nil } +func (f *fakeApp) HasChannel(string) bool { return false } +func (f *fakeApp) ProxyBin() string { return "proxy" } +func (f *fakeApp) CAHash() string { return "deadbeef" } +func (f *fakeApp) KeyMode() string { return "raw" } +func (f *fakeApp) VideoSelect() string { return "res=1280x720:for=best" } +func (f *fakeApp) AudioSelect() string { return "lang=en:for=best" } +func (f *fakeApp) RewriterName() string { return f.rewriter } +func (f *fakeApp) LiveWaitSeconds() int { return f.liveWait } +func (f *fakeApp) TSReadyBytes() int64 { return f.tsBytes } + +func register(t *testing.T, f *fakeApp) { + t.Helper() + app.Register(f.name, func() (app.App, error) { return f, nil }) +} + +func TestSettingsComeFromTheAppModule(t *testing.T) { + register(t, &fakeApp{name: "fakedash", rewriter: "fakedash", liveWait: 3, tsBytes: 111}) + got := settingsFor(db.Stream{App: "fakedash", MPD: "https://origin.test/manifest.mpd"}) + if got.Rewriter != "fakedash" { + t.Errorf("rewriter = %q, want fakedash", got.Rewriter) + } + if got.VideoSelect != "res=1280x720:for=best" || got.AudioSelect != "lang=en:for=best" { + t.Errorf("selectors = %q / %q", got.VideoSelect, got.AudioSelect) + } + if got.LiveWait != 3 || got.TSReadyBytes != 111 { + t.Errorf("liveWait=%d tsReadyBytes=%d", got.LiveWait, got.TSReadyBytes) + } + if got.IsHLS { + t.Error("an .mpd manifest is not HLS") + } +} + +// The stored row wins over the module, so an operator override sticks. +func TestStoredSettingsOverrideTheModule(t *testing.T) { + register(t, &fakeApp{name: "fakeoverride", rewriter: "fakeoverride"}) + got := settingsFor(db.Stream{ + App: "fakeoverride", + MPD: "https://origin.test/manifest.mpd", + VideoSelect: "res=1920x1080:for=best", + Rewriter: "none", + }) + if got.VideoSelect != "res=1920x1080:for=best" { + t.Errorf("video select = %q, want the stored value", got.VideoSelect) + } + if got.Rewriter != "none" { + t.Errorf("rewriter = %q, want the stored none", got.Rewriter) + } +} + +// An unknown app must not inherit any provider's preferences. +func TestUnknownAppGetsNeutralDefaults(t *testing.T) { + got := settingsFor(db.Stream{App: "nosuchapp", MPD: "https://origin.test/manifest.mpd"}) + if got.VideoSelect != "for=best" || got.AudioSelect != "for=best" { + t.Errorf("selectors = %q / %q, want for=best", got.VideoSelect, got.AudioSelect) + } + if got.Rewriter != "none" { + t.Errorf("rewriter = %q, want none", got.Rewriter) + } + if got.LiveWait != defaultLiveWait || got.TSReadyBytes != defaultTSReadyBytes { + t.Errorf("liveWait=%d tsReadyBytes=%d, want neutral defaults", got.LiveWait, got.TSReadyBytes) + } +} + +// HLS gets more buffering and a longer live wait, by manifest shape and not by +// matching a stream name. +func TestHLSDetectionByManifestShape(t *testing.T) { + got := settingsFor(db.Stream{App: "", MPD: "https://cdn.test/x/playlist-hls-dvr.m3u8"}) + if !got.IsHLS { + t.Fatal("an .m3u8 manifest should be detected as HLS") + } + if got.LiveWait != hlsLiveWait { + t.Errorf("liveWait = %d, want %d", got.LiveWait, hlsLiveWait) + } + if got.TSReadyBytes != hlsTSReadyBytes { + t.Errorf("tsReadyBytes = %d, want %d", got.TSReadyBytes, hlsTSReadyBytes) + } + // The old code keyed this off stream names like "tg4"/"cula"/"plus". + named := settingsFor(db.Stream{Name: "tg4-ioi", MPD: "https://origin.test/manifest.mpd"}) + if named.TSReadyBytes != defaultTSReadyBytes { + t.Error("buffering must not be chosen by stream name any more") + } +} + +// A module's rewriter is only used if it registered one under that name. +func TestRewriterLookupGatesTheRewrite(t *testing.T) { + register(t, &fakeApp{name: "fakeunregistered", rewriter: "fakeunregistered"}) + cfg := settingsFor(db.Stream{App: "fakeunregistered", MPD: "https://origin.test/manifest.mpd"}) + if _, needed := mpd.Lookup(cfg.Rewriter); needed { + t.Error("an unregistered rewriter name must not trigger a rewrite") + } + mpd.Register("fakeunregistered", func() mpd.Rewriter { return stubRewriter{} }) + if _, needed := mpd.Lookup(cfg.Rewriter); !needed { + t.Error("a registered rewriter must trigger a rewrite") + } +} + +type stubRewriter struct{} + +func (stubRewriter) Name() string { return "fakeunregistered" } +func (stubRewriter) Rewrite(in []byte, _ string) ([]byte, error) { return in, nil } diff --git a/apps/streamd/internal/supervisor/media.go b/apps/streamd/internal/supervisor/media.go new file mode 100644 index 0000000..b616146 --- /dev/null +++ b/apps/streamd/internal/supervisor/media.go @@ -0,0 +1,779 @@ +package supervisor + +import ( + "encoding/json" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "drmdecryption/apps/streamd/internal/db" + "drmdecryption/mpd" +) + +// Media runs N_m3u8DL-RE + ffmpeg HLS packager per stream into DataDir/www//. +type Media struct { + Store *db.Store + DataDir string + NRE string + FFmpeg string + MP4Decrypt string + Log *log.Logger + + mu sync.Mutex + procs map[int64]*streamProc + stopMon chan struct{} +} + +type streamProc struct { + id int64 + name string + nre *exec.Cmd + ffmpeg *exec.Cmd + started time.Time + workDir string + wwwDir string + tsPath string + hlsIndex string + nreLog *os.File + ffOutLog *os.File + ffErrLog *os.File + stopping bool + mpdProxy *mpd.LocalServer + nreDead bool + ffDead bool + // tsReadyBytes is how much muxed output to buffer before probing, from the + // stream's app module. + tsReadyBytes int64 +} + +// Options for constructing the media supervisor. +type Options struct { + Store *db.Store + DataDir string + NRE string + FFmpeg string + MP4Decrypt string + Log *log.Logger +} + +func NewMedia(opt Options) *Media { + lg := opt.Log + if lg == nil { + lg = log.Default() + } + m := &Media{ + Store: opt.Store, + DataDir: opt.DataDir, + NRE: opt.NRE, + FFmpeg: opt.FFmpeg, + MP4Decrypt: opt.MP4Decrypt, + Log: lg, + procs: map[int64]*streamProc{}, + stopMon: make(chan struct{}), + } + go m.monitorLoop() + return m +} + +func (m *Media) Close() { + close(m.stopMon) + m.mu.Lock() + ids := make([]int64, 0, len(m.procs)) + for id := range m.procs { + ids = append(ids, id) + } + m.mu.Unlock() + for _, id := range ids { + _ = m.Stop(id) + } +} + +func (m *Media) Start(id int64) error { + st, err := m.Store.GetStream(id) + if err != nil { + return err + } + if strings.TrimSpace(st.MPD) == "" || strings.TrimSpace(st.Key) == "" { + _ = m.Store.SetRuntime(id, db.RuntimePatch{ + Health: "down", + PlayPath: "/hls/" + st.Name + "/index.m3u8", + LastError: "missing mpd/key - capture required", + }) + return fmt.Errorf("missing mpd/key") + } + mpdURL := strings.TrimSpace(st.MPD) + if !strings.HasPrefix(strings.ToLower(mpdURL), "http://") && !strings.HasPrefix(strings.ToLower(mpdURL), "https://") { + _ = m.Store.SetRuntime(id, db.RuntimePatch{ + Health: "down", + PlayPath: "/hls/" + st.Name + "/index.m3u8", + LastError: "mpd must be an http(s) URL", + }) + return fmt.Errorf("invalid mpd url") + } + if m.NRE == "" || m.FFmpeg == "" { + _ = m.Store.SetRuntime(id, db.RuntimePatch{ + Health: "down", + PlayPath: "/hls/" + st.Name + "/index.m3u8", + LastError: "NRE or ffmpeg binary not configured", + }) + return fmt.Errorf("NRE or ffmpeg not configured") + } + + m.mu.Lock() + if _, exists := m.procs[id]; exists { + m.mu.Unlock() + return m.Restart(id) + } + m.mu.Unlock() + + _ = m.Store.SetRuntime(id, db.RuntimePatch{ + Health: "starting", + PlayPath: "/hls/" + st.Name + "/index.m3u8", + }) + + workDir := filepath.Join(m.DataDir, "work", st.Name) + wwwDir := filepath.Join(m.DataDir, "www", st.Name) + // Fresh dirs each start — leftover decrypted segments make NRE File.Move fail. + _ = os.RemoveAll(wwwDir) + _ = os.RemoveAll(workDir) + for _, d := range []string{workDir, wwwDir, filepath.Join(m.DataDir, "logs")} { + if err := os.MkdirAll(d, 0o755); err != nil { + return err + } + } + + keysFile := filepath.Join(workDir, "keys.txt") + keysBody := normalizeKeysFile(st.Key) + if err := os.WriteFile(keysFile, []byte(keysBody), 0o644); err != nil { + return err + } + + saveName := st.Name + tsPath := filepath.Join(workDir, saveName+".ts") + _ = os.Remove(tsPath) + + headers := headerMap(st.HeadersJSON) + manifestURL := st.MPD + cfg := settingsFor(st) + var mpdProxy *mpd.LocalServer + // The rewriter is named by the stream row or its app module and looked up in + // the mpd registry, so streamd needs no knowledge of any provider. + if rw, needed := mpd.Lookup(cfg.Rewriter); needed && strings.TrimSpace(st.MPD) != "" && !cfg.IsHLS { + srv, err := mpd.StartLocal(st.MPD, st.Key, headers, rw) + if err != nil { + _ = m.Store.SetRuntime(id, db.RuntimePatch{ + Health: "down", PlayPath: "/hls/" + st.Name + "/index.m3u8", + LastError: "mpd rewrite server: " + err.Error(), + }) + return err + } + mpdProxy = srv + manifestURL = srv.URL + m.logf("stream %s: rewritten MPD %s", st.Name, manifestURL) + } + + // Ad-stitched manifest rewrite + live pipe mux. + nreTmp := filepath.Join(workDir, "nre") + _ = os.MkdirAll(nreTmp, 0o755) + tsPath = filepath.Join(nreTmp, saveName+".ts") + _ = os.Remove(tsPath) + + liveWait := strconv.Itoa(cfg.LiveWait) + nreArgs := []string{ + manifestURL, + // Multi-KID SAMPLE-AES streams rainbow-decrypt with a single --key, so + // always feed every KID:KEY via --key-text-file. + "--key-text-file", keysFile, + "--live-real-time-merge", + "--live-pipe-mux", + "--mp4-real-time-decryption", + "--live-wait-time", liveWait, + "--ffmpeg-binary-path", m.FFmpeg, + "--save-name", saveName, + "--save-dir", nreTmp, + "--tmp-dir", nreTmp, + "--no-ansi-color", + "--log-level", "ERROR", + "-ss", "0", + } + // A synthetic manifest publishes exactly one video + audio pair, and an HLS + // master's variants rarely match a resolution/language filter, so in both + // cases take the best rendition instead of the configured selectors. + if mpdProxy != nil || cfg.IsHLS { + nreArgs = append(nreArgs, "-sv", "for=best", "-sa", "for=best") + } else { + nreArgs = append(nreArgs, "-sv", cfg.VideoSelect, "-sa", cfg.AudioSelect) + } + if m.MP4Decrypt != "" { + nreArgs = append(nreArgs, + "--decryption-engine", "MP4DECRYPT", + "--decryption-binary-path", m.MP4Decrypt, + ) + } + // Headers only needed when hitting upstream DAI directly (non-rewritten). + if mpdProxy == nil { + for _, h := range headerFlags(st.HeadersJSON) { + nreArgs = append(nreArgs, "-H", h) + } + } + _ = keysFile // kept on disk for debugging + + nreLogPath := filepath.Join(m.DataDir, "logs", st.Name+"-nre.log") + nreLog, err := os.OpenFile(nreLogPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + nreCmd := exec.Command(m.NRE, nreArgs...) + nreCmd.Stdout = nreLog + nreCmd.Stderr = nreLog + hideWindow(nreCmd) + // NRE (v0.6) resolves mp4decrypt via PATH / sibling of its .exe, and often + // ignores --decryption-binary-path for MP4DECRYPT. Match legacy: prepend + // dirs and best-effort copy beside NRE. + nreEnv := os.Environ() + pathPrepend := []string{} + if m.MP4Decrypt != "" { + pathPrepend = append(pathPrepend, filepath.Dir(m.MP4Decrypt)) + sibling := filepath.Join(filepath.Dir(m.NRE), filepath.Base(m.MP4Decrypt)) + if _, err := os.Stat(sibling); err != nil { + _ = copyFile(m.MP4Decrypt, sibling) + } + } + if m.NRE != "" { + pathPrepend = append(pathPrepend, filepath.Dir(m.NRE)) + } + if len(pathPrepend) > 0 { + sep := string(os.PathListSeparator) + nreEnv = prependPathEnv(nreEnv, strings.Join(pathPrepend, sep)) + } + nreCmd.Env = nreEnv + if err := nreCmd.Start(); err != nil { + _ = nreLog.Close() + if mpdProxy != nil { + mpdProxy.Close() + } + _ = m.Store.SetRuntime(id, db.RuntimePatch{Health: "down", LastError: err.Error(), PlayPath: "/hls/" + st.Name + "/index.m3u8"}) + return err + } + m.logf("stream %s: NRE pid=%d", st.Name, nreCmd.Process.Pid) + + sp := &streamProc{ + id: id, + name: st.Name, + nre: nreCmd, + started: time.Now(), + workDir: workDir, + wwwDir: wwwDir, + tsPath: tsPath, + hlsIndex: filepath.Join(wwwDir, "index.m3u8"), + nreLog: nreLog, + mpdProxy: mpdProxy, + + tsReadyBytes: cfg.TSReadyBytes, + } + + m.mu.Lock() + m.procs[id] = sp + m.mu.Unlock() + + go func() { + _ = nreCmd.Wait() + m.mu.Lock() + if cur, ok := m.procs[id]; ok && cur.nre == nreCmd { + cur.nreDead = true + } + m.mu.Unlock() + }() + + go m.bootstrapHLS(sp) + return nil +} + +func (m *Media) bootstrapHLS(sp *streamProc) { + deadline := time.Now().Add(3 * time.Minute) + nreDir := filepath.Dir(sp.tsPath) + minTS := sp.tsReadyBytes + for time.Now().Before(deadline) { + m.mu.Lock() + cur := m.procs[sp.id] + stopping := cur == nil || cur.stopping + nreDead := sp.nreDead + m.mu.Unlock() + if stopping { + return + } + if found := findGrowingTSMin(nreDir, sp.name, minTS); found != "" { + sp.tsPath = found + m.mu.Lock() + if cur, ok := m.procs[sp.id]; ok { + cur.tsPath = found + } + m.mu.Unlock() + break + } + if nreDead || (sp.nre.ProcessState != nil && sp.nre.ProcessState.Exited()) { + _ = m.Store.SetRuntime(sp.id, db.RuntimePatch{ + Health: "down", + PlayPath: "/hls/" + sp.name + "/index.m3u8", + LastError: "NRE exited before TS was ready — see logs/" + sp.name + "-nre.log", + }) + m.cleanupProc(sp.id) + return + } + time.Sleep(500 * time.Millisecond) + } + if st, err := os.Stat(sp.tsPath); err != nil || st.Size() < minTS { + _ = m.Store.SetRuntime(sp.id, db.RuntimePatch{ + Health: "down", + PlayPath: "/hls/" + sp.name + "/index.m3u8", + LastError: "timed out waiting for growing TS", + }) + _ = m.Stop(sp.id) + return + } + + if err := m.startFFmpegHLS(sp); err != nil { + _ = m.Store.SetRuntime(sp.id, db.RuntimePatch{ + Health: "down", LastError: err.Error(), PlayPath: "/hls/" + sp.name + "/index.m3u8", + }) + return + } + pid := int64(0) + if sp.nre != nil && sp.nre.Process != nil { + pid = int64(sp.nre.Process.Pid) + } + _ = m.Store.SetRuntime(sp.id, db.RuntimePatch{ + Health: "starting", + PID: pid, + PlayPath: "/hls/" + sp.name + "/index.m3u8", + }) +} + +func (m *Media) startFFmpegHLS(sp *streamProc) error { + ffOut := filepath.Join(m.DataDir, "logs", sp.name+"-ffmpeg.out.log") + ffErr := filepath.Join(m.DataDir, "logs", sp.name+"-ffmpeg.err.log") + outF, err := os.OpenFile(ffOut, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return err + } + errF, err := os.OpenFile(ffErr, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + _ = outF.Close() + return err + } + ffArgs := []string{ + "-hide_banner", "-loglevel", "warning", "-y", + "-follow", "1", + // Large probe: early pipe-mux bytes often lack clean audio config. + "-analyzeduration", "20M", + "-probesize", "10M", + "-fflags", "+genpts", + "-i", sp.tsPath, + "-map", "0", + "-c", "copy", + "-f", "hls", + "-hls_time", "4", + "-hls_list_size", "15", + "-hls_flags", "delete_segments+append_list+omit_endlist+independent_segments", + "-hls_segment_type", "mpegts", + "-hls_segment_filename", filepath.Join(sp.wwwDir, "seg_%05d.ts"), + sp.hlsIndex, + } + ffCmd := exec.Command(m.FFmpeg, ffArgs...) + ffCmd.Stdout = outF + ffCmd.Stderr = errF + hideWindow(ffCmd) + if err := ffCmd.Start(); err != nil { + _ = outF.Close() + _ = errF.Close() + return err + } + m.logf("stream %s: ffmpeg HLS pid=%d → %s", sp.name, ffCmd.Process.Pid, sp.hlsIndex) + + m.mu.Lock() + if cur, ok := m.procs[sp.id]; ok && !cur.stopping { + if cur.ffOutLog != nil { + _ = cur.ffOutLog.Close() + } + if cur.ffErrLog != nil { + _ = cur.ffErrLog.Close() + } + cur.ffmpeg = ffCmd + cur.ffOutLog = outF + cur.ffErrLog = errF + cur.ffDead = false + } else { + m.mu.Unlock() + _ = killProcess(ffCmd) + _ = outF.Close() + _ = errF.Close() + return fmt.Errorf("stream stopped before ffmpeg attach") + } + m.mu.Unlock() + + go func() { + _ = ffCmd.Wait() + m.mu.Lock() + if cur, ok := m.procs[sp.id]; ok && cur.ffmpeg == ffCmd { + cur.ffDead = true + } + m.mu.Unlock() + }() + return nil +} + +func (m *Media) Stop(id int64) error { + m.mu.Lock() + sp, ok := m.procs[id] + if ok { + sp.stopping = true + } + m.mu.Unlock() + if !ok { + st, _ := m.Store.GetStream(id) + name := fmt.Sprintf("%d", id) + if st.Name != "" { + name = st.Name + } + _ = m.Store.SetRuntime(id, db.RuntimePatch{Health: "stopped", PlayPath: "/hls/" + name + "/index.m3u8"}) + return nil + } + nrePID, ffPID := 0, 0 + if sp.ffmpeg != nil && sp.ffmpeg.Process != nil { + ffPID = sp.ffmpeg.Process.Pid + _ = killProcess(sp.ffmpeg) + } + if sp.nre != nil && sp.nre.Process != nil { + nrePID = sp.nre.Process.Pid + _ = killProcess(sp.nre) + } + if sp.mpdProxy != nil { + sp.mpdProxy.Close() + sp.mpdProxy = nil + } + waitPIDsGone([]int{nrePID, ffPID}, 5*time.Second) + m.cleanupProc(id) + _ = m.Store.SetRuntime(id, db.RuntimePatch{ + Health: "stopped", + PlayPath: "/hls/" + sp.name + "/index.m3u8", + }) + m.logf("stream %s: stopped", sp.name) + return nil +} + +func (m *Media) Restart(id int64) error { + m.logf("stream id=%d: hard restart (kill worker, fresh start)", id) + _ = m.Stop(id) + // Give Windows time to release file locks on work/www before Start wipes them. + time.Sleep(1 * time.Second) + return m.Start(id) +} + +func (m *Media) cleanupProc(id int64) { + m.mu.Lock() + defer m.mu.Unlock() + sp, ok := m.procs[id] + if !ok { + return + } + if sp.nreLog != nil { + _ = sp.nreLog.Close() + } + if sp.ffOutLog != nil { + _ = sp.ffOutLog.Close() + } + if sp.ffErrLog != nil { + _ = sp.ffErrLog.Close() + } + delete(m.procs, id) +} + +func (m *Media) monitorLoop() { + t := time.NewTicker(5 * time.Second) + defer t.Stop() + for { + select { + case <-m.stopMon: + return + case <-t.C: + m.mu.Lock() + ids := make([]int64, 0, len(m.procs)) + for id := range m.procs { + ids = append(ids, id) + } + m.mu.Unlock() + for _, id := range ids { + m.refreshHealth(id) + } + } + } +} + +func (m *Media) refreshHealth(id int64) { + m.mu.Lock() + sp, ok := m.procs[id] + m.mu.Unlock() + if !ok || sp.stopping { + return + } + + m.mu.Lock() + nreDead := sp.nreDead + ffDead := sp.ffDead + m.mu.Unlock() + + playPath := "/hls/" + sp.name + "/index.m3u8" + uptime := time.Since(sp.started).Seconds() + pid := int64(0) + if sp.nre != nil && sp.nre.Process != nil { + pid = int64(sp.nre.Process.Pid) + } + + if nreDead { + _ = m.Store.SetRuntime(id, db.RuntimePatch{ + Health: "down", PID: 0, PlayPath: playPath, UptimeS: uptime, + LastError: "NRE process exited — see logs/" + sp.name + "-nre.log", + }) + m.cleanupProc(id) + return + } + + fi, err := os.Stat(sp.hlsIndex) + if err != nil { + _ = m.Store.SetRuntime(id, db.RuntimePatch{ + Health: "starting", PID: pid, PlayPath: playPath, UptimeS: uptime, + LastError: "waiting for HLS playlist", + }) + if ffDead { + // restart ffmpeg packager if TS still growing + go m.restartFFmpeg(sp) + } + return + } + age := time.Since(fi.ModTime()).Seconds() + health := "ok" + lastErr := "" + if age > 45 { + health = "down" + lastErr = fmt.Sprintf("playlist stale (%.0fs)", age) + } + _ = m.Store.SetRuntime(id, db.RuntimePatch{ + Health: health, PID: pid, PlayPath: playPath, UptimeS: uptime, + PlaylistAgeS: age, LastError: lastErr, + }) + if ffDead && health != "down" { + go m.restartFFmpeg(sp) + } +} + +func (m *Media) restartFFmpeg(sp *streamProc) { + m.mu.Lock() + cur, ok := m.procs[sp.id] + if !ok || cur.stopping || !cur.ffDead { + m.mu.Unlock() + return + } + // Claim the restart slot so monitorLoop does not spawn duplicates. + cur.ffDead = false + if cur.ffmpeg != nil { + _ = killProcess(cur.ffmpeg) + } + tsPath := cur.tsPath + m.mu.Unlock() + + minTS := sp.tsReadyBytes + if st, err := os.Stat(tsPath); err != nil || st.Size() < minTS { + m.mu.Lock() + if cur, ok := m.procs[sp.id]; ok { + cur.ffDead = true + } + m.mu.Unlock() + return + } + m.logf("stream %s: restarting ffmpeg HLS packager", sp.name) + if err := m.startFFmpegHLS(sp); err != nil { + m.logf("stream %s: ffmpeg restart failed: %v", sp.name, err) + m.mu.Lock() + if cur, ok := m.procs[sp.id]; ok { + cur.ffDead = true + } + m.mu.Unlock() + } +} + +func (m *Media) logf(format string, args ...any) { + m.Log.Printf("supervisor: "+format, args...) +} + +func headerMap(headersJSON string) map[string]string { + headersJSON = strings.TrimSpace(headersJSON) + if headersJSON == "" || headersJSON == "{}" { + return nil + } + var m map[string]string + if err := json.Unmarshal([]byte(headersJSON), &m); err != nil { + return nil + } + out := map[string]string{} + for k, v := range m { + k = strings.TrimSpace(k) + v = strings.TrimSpace(v) + if k != "" && v != "" { + out[k] = v + } + } + return out +} + +func headerFlags(headersJSON string) []string { + m := headerMap(headersJSON) + out := make([]string, 0, len(m)) + for k, v := range m { + out = append(out, k+": "+v) + } + return out +} + +// normalizeKeysFile turns a DB key field (one or many KID:KEY lines, or +// comma/semicolon separated) into an NRE --key-text-file body. +func normalizeKeysFile(raw string) string { + raw = strings.ReplaceAll(raw, ",", "\n") + raw = strings.ReplaceAll(raw, ";", "\n") + var lines []string + seen := map[string]bool{} + for _, line := range strings.Split(raw, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.Count(line, ":") != 1 { + continue + } + if seen[line] { + continue + } + seen[line] = true + lines = append(lines, line) + } + if len(lines) == 0 { + return strings.TrimSpace(raw) + "\n" + } + return strings.Join(lines, "\n") + "\n" +} + +func killProcess(cmd *exec.Cmd) error { + if cmd == nil || cmd.Process == nil { + return nil + } + pid := cmd.Process.Pid + if runtime.GOOS == "windows" { + _ = exec.Command("taskkill", "/T", "/F", "/PID", strconv.Itoa(pid)).Run() + return nil + } + return cmd.Process.Kill() +} + +func prependPathEnv(env []string, prefix string) []string { + if prefix == "" { + return env + } + out := make([]string, 0, len(env)+1) + found := false + for _, e := range env { + if len(e) >= 5 && strings.EqualFold(e[:5], "PATH=") { + out = append(out, "PATH="+prefix+string(os.PathListSeparator)+e[5:]) + found = true + continue + } + out = append(out, e) + } + if !found { + out = append(out, "PATH="+prefix) + } + return out +} + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755) + if err != nil { + return err + } + defer out.Close() + if _, err := io.Copy(out, in); err != nil { + return err + } + return out.Close() +} + +// findGrowingTS locates NRE's live-pipe-mux output. Depending on version/flags +// it may be saveName.ts or saveName..ts (e.g. test.en.ts). +func waitPIDsGone(pids []int, timeout time.Duration) { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + alive := false + for _, pid := range pids { + if pid <= 0 { + continue + } + if !processDead(pid) { + alive = true + break + } + } + if !alive { + return + } + time.Sleep(100 * time.Millisecond) + } +} + +func findGrowingTS(dir, saveName string) string { + return findGrowingTSMin(dir, saveName, 256*1024) +} + +func findGrowingTSMin(dir, saveName string, minSize int64) string { + if minSize <= 0 { + minSize = 256 * 1024 + } + candidates := []string{ + filepath.Join(dir, saveName+".ts"), + } + if ents, err := os.ReadDir(dir); err == nil { + prefix := saveName + "." + for _, e := range ents { + if e.IsDir() { + continue + } + name := e.Name() + if !strings.HasSuffix(strings.ToLower(name), ".ts") { + continue + } + if name == saveName+".ts" || strings.HasPrefix(name, prefix) { + candidates = append(candidates, filepath.Join(dir, name)) + } + } + } + var best string + var bestSize int64 + for _, p := range candidates { + st, err := os.Stat(p) + if err != nil || st.Size() < minSize { + continue + } + if st.Size() > bestSize { + bestSize = st.Size() + best = p + } + } + return best +} diff --git a/apps/streamd/internal/supervisor/proc_unix.go b/apps/streamd/internal/supervisor/proc_unix.go new file mode 100644 index 0000000..266175f --- /dev/null +++ b/apps/streamd/internal/supervisor/proc_unix.go @@ -0,0 +1,20 @@ +//go:build !windows + +package supervisor + +import ( + "os" + "os/exec" + "syscall" +) + +func hideWindow(cmd *exec.Cmd) {} + +func processDead(pid int) bool { + p, err := os.FindProcess(pid) + if err != nil { + return true + } + err = p.Signal(syscall.Signal(0)) + return err != nil +} diff --git a/apps/streamd/internal/supervisor/proc_windows.go b/apps/streamd/internal/supervisor/proc_windows.go new file mode 100644 index 0000000..4de3aff --- /dev/null +++ b/apps/streamd/internal/supervisor/proc_windows.go @@ -0,0 +1,35 @@ +//go:build windows + +package supervisor + +import ( + "os" + "os/exec" + "syscall" +) + +func hideWindow(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{ + HideWindow: true, + CreationFlags: 0x08000000, // CREATE_NO_WINDOW + } +} + +func processDead(pid int) bool { + p, err := os.FindProcess(pid) + if err != nil { + return true + } + // On Windows, FindProcess always succeeds; OpenProcess is needed. + // tasklist-style: try duplicate handle via Signal is unsupported. + // Use Windows API indirectly: if Wait with timeout 0 isn't available, + // check via tasklist is heavy — use syscall OpenProcess. + const PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 + h, err := syscall.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + return true + } + _ = p + syscall.CloseHandle(h) + return false +} diff --git a/apps/streamd/internal/supervisor/stub.go b/apps/streamd/internal/supervisor/stub.go new file mode 100644 index 0000000..a847894 --- /dev/null +++ b/apps/streamd/internal/supervisor/stub.go @@ -0,0 +1,71 @@ +package supervisor + +import ( + "fmt" + "sync" + "time" + + "drmdecryption/apps/streamd/internal/db" +) + +// Stub marks streams as starting/stopped without spawning media workers. +// Replaced by a real NRE/ffmpeg supervisor in Phase 2. +type Stub struct { + Store *db.Store + mu sync.Mutex + pids map[int64]int +} + +func NewStub(store *db.Store) *Stub { + return &Stub{Store: store, pids: map[int64]int{}} +} + +func (s *Stub) Start(id int64) error { + s.mu.Lock() + defer s.mu.Unlock() + st, err := s.Store.GetStream(id) + if err != nil { + return err + } + if st.MPD == "" || st.Key == "" { + _, err = s.Store.DB.Exec(` +INSERT INTO stream_runtime(stream_id,health,play_path,last_error,updated_at) +VALUES(?,?,?,?,?) +ON CONFLICT(stream_id) DO UPDATE SET health=excluded.health, last_error=excluded.last_error, updated_at=excluded.updated_at`, + id, "down", "/hls/"+st.Name+"/index.m3u8", "missing mpd/key - capture required", time.Now().UTC().Format(time.RFC3339)) + return err + } + pid := 10000 + int(id) + s.pids[id] = pid + _, err = s.Store.DB.Exec(` +INSERT INTO stream_runtime(stream_id,health,pid,play_path,uptime_s,playlist_age_s,last_error,updated_at) +VALUES(?,?,?,?,0,1,'',?) +ON CONFLICT(stream_id) DO UPDATE SET + health=excluded.health, pid=excluded.pid, play_path=excluded.play_path, + playlist_age_s=excluded.playlist_age_s, last_error='', updated_at=excluded.updated_at`, + id, "ok", pid, "/hls/"+st.Name+"/index.m3u8", time.Now().UTC().Format(time.RFC3339)) + return err +} + +func (s *Stub) Stop(id int64) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.pids, id) + st, err := s.Store.GetStream(id) + name := fmt.Sprintf("%d", id) + if err == nil { + name = st.Name + } + _, err = s.Store.DB.Exec(` +INSERT INTO stream_runtime(stream_id,health,pid,play_path,uptime_s,bitrate_mbps,playlist_age_s,last_error,updated_at) +VALUES(?,?,0,?,0,0,0,'',?) +ON CONFLICT(stream_id) DO UPDATE SET + health='stopped', pid=0, uptime_s=0, bitrate_mbps=0, playlist_age_s=0, last_error='', updated_at=excluded.updated_at`, + id, "stopped", "/hls/"+name+"/index.m3u8", time.Now().UTC().Format(time.RFC3339)) + return err +} + +func (s *Stub) Restart(id int64) error { + _ = s.Stop(id) + return s.Start(id) +} diff --git a/apps/streamd/internal/ui/app.css b/apps/streamd/internal/ui/app.css new file mode 100644 index 0000000..778d890 --- /dev/null +++ b/apps/streamd/internal/ui/app.css @@ -0,0 +1,106 @@ +:root { + --bg: #0f1115; + --card: #171a21; + --border: #2a303c; + --text: #e8eaed; + --muted: #9aa0a6; + --accent: #7aa2ff; + --ok: #3dd68c; + --degraded: #f5a524; + --down: #f07178; + --stopped: #6b7280; +} +* { box-sizing: border-box; } +body { + margin: 0; + font: 14px/1.45 system-ui, Segoe UI, sans-serif; + background: var(--bg); + color: var(--text); +} +header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 12px 20px; + border-bottom: 1px solid var(--border); +} +h1 { margin: 0; font-size: 18px; font-weight: 600; } +h2 { margin: 0 0 12px; font-size: 15px; font-weight: 600; } +.header-right { display: flex; gap: 16px; align-items: center; } +.muted { color: var(--muted); } +.card { + margin: 16px 20px; + padding: 16px; + background: var(--card); + border: 1px solid var(--border); + border-radius: 8px; +} +.grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 10px 14px; +} +.span2 { grid-column: span 2; } +label { display: flex; flex-direction: column; gap: 4px; font-size: 12px; color: var(--muted); } +input, select, button, textarea { + font: inherit; + color: var(--text); + background: #0c0e12; + border: 1px solid var(--border); + border-radius: 6px; + padding: 7px 9px; +} +textarea { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 12px; + line-height: 1.4; + resize: vertical; + min-height: 4.5em; + white-space: pre; +} +button { + background: #243044; + border-color: #334155; + cursor: pointer; +} +button:hover { border-color: var(--accent); } +button.ghost { background: transparent; } +button.danger { border-color: #7f1d1d; color: #fecaca; } +.actions { display: flex; gap: 8px; align-items: end; } +.token-box input { width: 180px; } +.row-between { display: flex; justify-content: space-between; align-items: center; } +.table-wrap { overflow-x: auto; } +table { width: 100%; border-collapse: collapse; } +th, td { text-align: left; padding: 8px 6px; border-bottom: 1px solid var(--border); vertical-align: middle; } +th { color: var(--muted); font-weight: 500; font-size: 12px; } +.pill { + display: inline-block; + padding: 2px 8px; + border-radius: 999px; + font-size: 12px; + background: #222; +} +.pill.ok { color: var(--ok); } +.pill.degraded { color: var(--degraded); } +.pill.down, .pill.starting { color: var(--down); } +.pill.stopped { color: var(--stopped); } +.pill.claim { color: #c4b5fd; background: #1f1635; } +.row-actions { display: flex; flex-wrap: wrap; gap: 6px; } +.row-actions button { padding: 4px 8px; font-size: 12px; } +.msg { min-height: 1.2em; margin: 8px 0 0; } +.msg.err { color: var(--down); } +a { color: var(--accent); } +dialog { + border: 1px solid var(--border); + background: var(--card); + color: var(--text); + border-radius: 8px; + padding: 16px; + min-width: min(560px, 92vw); +} +dialog::backdrop { background: rgba(0,0,0,.55); } +@media (max-width: 720px) { + .grid { grid-template-columns: 1fr; } + .span2 { grid-column: span 1; } + header { flex-direction: column; align-items: flex-start; gap: 8px; } +} diff --git a/apps/streamd/internal/ui/app.js b/apps/streamd/internal/ui/app.js new file mode 100644 index 0000000..d42c50e --- /dev/null +++ b/apps/streamd/internal/ui/app.js @@ -0,0 +1,286 @@ +(() => { + const tokenEl = document.getElementById("token"); + const rowsEl = document.getElementById("rows"); + const listMsg = document.getElementById("list-msg"); + const formMsg = document.getElementById("form-msg"); + const uptimeEl = document.getElementById("uptime"); + const agentsEl = document.getElementById("agents-status"); + const dlg = document.getElementById("edit-dlg"); + const editForm = document.getElementById("edit-form"); + + tokenEl.value = localStorage.getItem("streamd_token") || ""; + tokenEl.addEventListener("change", () => { + localStorage.setItem("streamd_token", tokenEl.value.trim()); + connectUIWS(); + }); + + function token() { + return tokenEl.value.trim(); + } + + function headers(json) { + const h = {}; + if (json) h["Content-Type"] = "application/json"; + const t = token(); + if (t) h["Authorization"] = "Bearer " + t; + return h; + } + + async function api(path, opts = {}) { + const res = await fetch(path, opts); + const text = await res.text(); + let data = null; + try { data = text ? JSON.parse(text) : null; } catch { data = { raw: text }; } + if (!res.ok) { + const msg = (data && data.error) || res.statusText || "request failed"; + throw new Error(msg); + } + return data; + } + + function age(s) { + if (s == null || s === 0) return "—"; + return Number(s).toFixed(1) + "s"; + } + + function claimLabel(s) { + if (!s.claimed_by) return '—'; + let exp = ""; + if (s.claim_expires_at) { + const ms = Date.parse(s.claim_expires_at) - Date.now(); + if (!Number.isNaN(ms)) { + exp = ms > 0 ? ` (${Math.ceil(ms / 1000)}s)` : " (expired)"; + } + } + return `${esc(s.claimed_by)}${esc(exp)}`; + } + + function render(streams) { + if (!streams.length) { + rowsEl.innerHTML = ""; + listMsg.textContent = "No streams yet. Add one above."; + return; + } + listMsg.textContent = streams.length + " stream(s)"; + rowsEl.innerHTML = streams.map((s) => { + const need = !s.mpd || !s.key ? " needs capture" : ""; + return ` + ${esc(s.name)}
${esc(s.title || "")}${need}
+ ${esc(s.app)} / ${esc(s.channel)} + ${esc(s.health || "stopped")} + ${claimLabel(s)} + ${age(s.playlist_age_s)} + hls + + + + + + ${s.claimed_by ? `` : ""} + + + `; + }).join(""); + } + + function esc(v) { + return String(v ?? "").replace(/[&<>"']/g, (c) => ({ + "&": "&", "<": "<", ">": ">", '"': """, "'": "'" + }[c])); + } + + function renderAgents(list) { + if (!agentsEl) return; + if (!list || !list.length) { + agentsEl.textContent = "agents: none connected"; + return; + } + agentsEl.textContent = "agents: " + list.map((a) => a.agent_id).join(", "); + } + + async function refresh() { + try { + const [health, list, agents] = await Promise.all([ + api("/api/health"), + api("/api/streams"), + api("/api/agents").catch(() => ({ agents: [] })), + ]); + uptimeEl.textContent = "up " + Math.floor(health.uptime_s || 0) + "s"; + render(list.streams || []); + renderAgents(agents.agents || []); + listMsg.classList.remove("err"); + } catch (e) { + listMsg.textContent = e.message; + listMsg.classList.add("err"); + } + } + + // App/channel pickers come from the binary's compiled-in modules, so the UI + // never hardcodes a provider. + const appSel = document.getElementById("app-select"); + const chanSel = document.getElementById("channel-select"); + let apps = []; + + function fillChannels() { + const a = apps.find((x) => x.name === appSel.value); + chanSel.innerHTML = ""; + if (!a || !a.channels.length) { + const o = document.createElement("option"); + o.value = ""; + o.textContent = a ? "(no module.yaml — no channels)" : "(no apps)"; + chanSel.appendChild(o); + return; + } + for (const ch of a.channels) { + const o = document.createElement("option"); + o.value = ch.id; + o.textContent = ch.label && ch.label !== ch.id ? ch.id + " — " + ch.label : ch.id; + chanSel.appendChild(o); + } + } + + async function loadApps() { + try { + const data = await api("/api/apps"); + apps = data.apps || []; + } catch (e) { + apps = []; + } + appSel.innerHTML = ""; + if (!apps.length) { + const o = document.createElement("option"); + o.value = ""; + o.textContent = "(no app modules compiled in)"; + appSel.appendChild(o); + } + for (const a of apps) { + const o = document.createElement("option"); + o.value = a.name; + o.textContent = a.name; + appSel.appendChild(o); + } + fillChannels(); + } + + appSel.addEventListener("change", fillChannels); + loadApps(); + + document.getElementById("refresh").addEventListener("click", refresh); + + document.getElementById("create-form").addEventListener("submit", async (ev) => { + ev.preventDefault(); + formMsg.textContent = ""; + formMsg.classList.remove("err"); + const fd = new FormData(ev.target); + const body = Object.fromEntries(fd.entries()); + try { + await api("/api/streams", { + method: "POST", + headers: headers(true), + body: JSON.stringify(body), + }); + ev.target.reset(); + ev.target.headers_json.value = "{}"; + fillChannels(); + formMsg.textContent = "Created."; + refresh(); + } catch (e) { + formMsg.textContent = e.message; + formMsg.classList.add("err"); + } + }); + + rowsEl.addEventListener("click", async (ev) => { + const btn = ev.target.closest("button[data-act]"); + if (!btn) return; + const id = btn.dataset.id; + const act = btn.dataset.act; + try { + if (act === "delete") { + if (!confirm("Delete stream #" + id + "?")) return; + await api("/api/streams/" + id, { method: "DELETE", headers: headers(false) }); + } else if (act === "unclaim") { + await api("/api/streams/" + id + "/claim/release", { + method: "POST", + headers: headers(true), + body: JSON.stringify({ agent_id: "" }), + }); + } else if (act === "edit") { + const st = await api("/api/streams/" + id); + editForm.id.value = st.id; + editForm.title.value = st.title || ""; + editForm.channel.value = st.channel || ""; + editForm.mpd.value = st.mpd || ""; + editForm.key.value = st.key || ""; + editForm.headers_json.value = st.headers_json || "{}"; + dlg.showModal(); + return; + } else { + await api("/api/streams/" + id + "/" + act, { + method: "POST", + headers: headers(false), + }); + } + refresh(); + } catch (e) { + listMsg.textContent = e.message; + listMsg.classList.add("err"); + } + }); + + document.getElementById("edit-cancel").addEventListener("click", () => dlg.close()); + + editForm.addEventListener("submit", async (ev) => { + ev.preventDefault(); + const id = editForm.id.value; + const body = { + title: editForm.title.value, + channel: editForm.channel.value, + mpd: editForm.mpd.value, + key: editForm.key.value, + headers_json: editForm.headers_json.value, + }; + try { + await api("/api/streams/" + id, { + method: "PATCH", + headers: headers(true), + body: JSON.stringify(body), + }); + dlg.close(); + refresh(); + } catch (e) { + alert(e.message); + } + }); + + let uiWS = null; + let uiWSTimer = null; + function connectUIWS() { + if (uiWS) { + try { uiWS.close(); } catch {} + uiWS = null; + } + const proto = location.protocol === "https:" ? "wss:" : "ws:"; + const ws = new WebSocket(proto + "//" + location.host + "/ws/ui"); + uiWS = ws; + ws.onmessage = (ev) => { + try { + const msg = JSON.parse(ev.data); + if (msg.type === "agents" || msg.type === "agent_online" || msg.type === "agent_offline") { + renderAgents(msg.agents || []); + } + if (msg.type === "claims_changed" || msg.type === "agent_offline") { + refresh(); + } + } catch {} + }; + ws.onclose = () => { + clearTimeout(uiWSTimer); + uiWSTimer = setTimeout(connectUIWS, 3000); + }; + } + + refresh(); + connectUIWS(); + setInterval(refresh, 5000); +})(); diff --git a/apps/streamd/internal/ui/index.html b/apps/streamd/internal/ui/index.html new file mode 100644 index 0000000..6418772 --- /dev/null +++ b/apps/streamd/internal/ui/index.html @@ -0,0 +1,84 @@ + + + + + + streamd + + + +
+

streamd

+
+ + +
+
+ +
+

Add stream

+
+ + + + + + + +
+ +
+
+

+
+ +
+
+

Streams

+
+ agents: — + +
+
+
+ + + + + + + + + + + + + +
nameapp / channelhealthclaimageplayactions
+
+

Loading…

+
+ + +
+

Edit stream

+ + + + + + +
+ + +
+
+ + + + + diff --git a/apps/streamd/internal/ui/ui.go b/apps/streamd/internal/ui/ui.go new file mode 100644 index 0000000..0fcaf03 --- /dev/null +++ b/apps/streamd/internal/ui/ui.go @@ -0,0 +1,34 @@ +package ui + +import ( + "embed" + "io/fs" + "net/http" +) + +//go:embed index.html app.css app.js +var files embed.FS + +func Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" && r.URL.Path != "/index.html" { + http.NotFound(w, r) + return + } + b, err := files.ReadFile("index.html") + if err != nil { + http.Error(w, err.Error(), 500) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write(b) + }) +} + +func Static() http.Handler { + sub, err := fs.Sub(files, ".") + if err != nil { + panic(err) + } + return http.FileServer(http.FS(sub)) +} diff --git a/apps/streamd/internal/ws/hub.go b/apps/streamd/internal/ws/hub.go new file mode 100644 index 0000000..1bec51f --- /dev/null +++ b/apps/streamd/internal/ws/hub.go @@ -0,0 +1,350 @@ +package ws + +import ( + "encoding/json" + "log" + "net/http" + "strings" + "sync" + "time" + + "github.com/gorilla/websocket" + + "drmdecryption/apps/streamd/internal/db" +) + +const ( + agentHeartbeatEvery = 20 * time.Second + agentStaleAfter = 45 * time.Second + claimRenewTTL = 90 * time.Second + writeWait = 10 * time.Second + pongWait = 60 * time.Second +) + +var upgrader = websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { return true }, +} + +// Hub tracks agent WebSocket sessions and UI listeners. +type Hub struct { + Store *db.Store + Token string + Log *log.Logger + + mu sync.Mutex + agents map[string]*agentConn // agent_id -> conn + uis map[*websocket.Conn]struct{} +} + +type agentConn struct { + conn *websocket.Conn + agentID string + lastSeen time.Time + send chan []byte +} + +type inbound struct { + Type string `json:"type"` + AgentID string `json:"agent_id,omitempty"` +} + +type outbound struct { + Type string `json:"type"` + AgentID string `json:"agent_id,omitempty"` + OK bool `json:"ok,omitempty"` + Error string `json:"error,omitempty"` + Agents []any `json:"agents,omitempty"` + Message string `json:"message,omitempty"` +} + +func NewHub(store *db.Store, token string, lg *log.Logger) *Hub { + if lg == nil { + lg = log.Default() + } + return &Hub{ + Store: store, + Token: token, + Log: lg, + agents: map[string]*agentConn{}, + uis: map[*websocket.Conn]struct{}{}, + } +} + +func (h *Hub) Mount(mux *http.ServeMux) { + mux.HandleFunc("/ws/agent", h.handleAgent) + mux.HandleFunc("/ws/ui", h.handleUI) + mux.HandleFunc("/api/agents", h.handleAgentsAPI) +} + +// RunExpireLoop periodically drops expired claims and dead agent sessions. +func (h *Hub) RunExpireLoop(stop <-chan struct{}) { + t := time.NewTicker(10 * time.Second) + defer t.Stop() + for { + select { + case <-stop: + return + case <-t.C: + h.tick() + } + } +} + +func (h *Hub) tick() { + n, err := h.Store.ExpireClaims() + if err != nil { + h.Log.Printf("ws: expire claims: %v", err) + } else if n > 0 { + h.Log.Printf("ws: expired %d stale claim(s)", n) + h.broadcastUI(outbound{Type: "claims_changed", Message: "expired"}) + } + + now := time.Now().UTC() + var stale []string + h.mu.Lock() + for id, a := range h.agents { + if now.Sub(a.lastSeen) > agentStaleAfter { + stale = append(stale, id) + } + } + h.mu.Unlock() + for _, id := range stale { + h.Log.Printf("ws: agent %s heartbeat stale — releasing claims", id) + h.dropAgent(id, true) + } +} + +func (h *Hub) authToken(r *http.Request) bool { + if h.Token == "" { + return true + } + tok := r.URL.Query().Get("token") + if tok == "" { + auth := r.Header.Get("Authorization") + tok = strings.TrimPrefix(auth, "Bearer ") + } + if tok == "" { + tok = r.Header.Get("X-Streamd-Token") + } + return tok == h.Token +} + +func (h *Hub) handleAgentsAPI(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed) + return + } + writeJSON(w, map[string]any{"agents": h.snapshotAgents()}) +} + +func (h *Hub) handleUI(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + h.mu.Lock() + h.uis[conn] = struct{}{} + h.mu.Unlock() + defer func() { + h.mu.Lock() + delete(h.uis, conn) + h.mu.Unlock() + _ = conn.Close() + }() + + _ = conn.SetReadDeadline(time.Now().Add(pongWait)) + conn.SetPongHandler(func(string) error { + _ = conn.SetReadDeadline(time.Now().Add(pongWait)) + return nil + }) + // Push initial agent list. + h.sendJSON(conn, outbound{Type: "agents", Agents: h.snapshotAgents()}) + for { + if _, _, err := conn.ReadMessage(); err != nil { + return + } + } +} + +func (h *Hub) handleAgent(w http.ResponseWriter, r *http.Request) { + if !h.authToken(r) { + http.Error(w, `{"error":"unauthorized"}`, http.StatusUnauthorized) + return + } + agentID := strings.TrimSpace(r.URL.Query().Get("agent_id")) + if agentID == "" { + http.Error(w, `{"error":"agent_id required"}`, http.StatusBadRequest) + return + } + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + ac := &agentConn{ + conn: conn, + agentID: agentID, + lastSeen: time.Now().UTC(), + send: make(chan []byte, 8), + } + h.mu.Lock() + if old, ok := h.agents[agentID]; ok { + close(old.send) + _ = old.conn.Close() + } + h.agents[agentID] = ac + h.mu.Unlock() + h.Log.Printf("ws: agent connected id=%s", agentID) + h.broadcastUI(outbound{Type: "agent_online", AgentID: agentID, Agents: h.snapshotAgents()}) + + go h.agentWriter(ac) + + defer h.dropAgent(agentID, true) + + _ = conn.SetReadDeadline(time.Now().Add(pongWait)) + conn.SetPongHandler(func(string) error { + _ = conn.SetReadDeadline(time.Now().Add(pongWait)) + return nil + }) + + // Greeting + first renew. + h.renew(agentID) + h.queue(ac, outbound{Type: "hello", AgentID: agentID, OK: true}) + + for { + _, data, err := conn.ReadMessage() + if err != nil { + return + } + _ = conn.SetReadDeadline(time.Now().Add(pongWait)) + var msg inbound + if err := json.Unmarshal(data, &msg); err != nil { + continue + } + switch strings.ToLower(msg.Type) { + case "heartbeat", "ping", "hello": + h.mu.Lock() + if cur, ok := h.agents[agentID]; ok { + cur.lastSeen = time.Now().UTC() + } + h.mu.Unlock() + n, err := h.renew(agentID) + if err != nil { + h.queue(ac, outbound{Type: "heartbeat_ack", OK: false, Error: err.Error()}) + continue + } + h.queue(ac, outbound{Type: "heartbeat_ack", OK: true, Message: "renewed", AgentID: agentID}) + if n > 0 { + h.broadcastUI(outbound{Type: "claims_changed", AgentID: agentID, Message: "renewed"}) + } + case "release_all": + _, _ = h.Store.ReleaseAgentClaims(agentID) + h.broadcastUI(outbound{Type: "claims_changed", AgentID: agentID, Message: "released"}) + h.queue(ac, outbound{Type: "release_ack", OK: true}) + } + } +} + +func (h *Hub) renew(agentID string) (int64, error) { + return h.Store.RenewAgentClaims(agentID, claimRenewTTL) +} + +func (h *Hub) dropAgent(agentID string, releaseClaims bool) { + h.mu.Lock() + ac, ok := h.agents[agentID] + if ok { + delete(h.agents, agentID) + } + h.mu.Unlock() + if !ok { + return + } + close(ac.send) + _ = ac.conn.Close() + if releaseClaims { + n, err := h.Store.ReleaseAgentClaims(agentID) + if err != nil { + h.Log.Printf("ws: release claims for %s: %v", agentID, err) + } else if n > 0 { + h.Log.Printf("ws: released %d claim(s) for disconnected agent %s", n, agentID) + } + } + h.Log.Printf("ws: agent disconnected id=%s", agentID) + h.broadcastUI(outbound{Type: "agent_offline", AgentID: agentID, Agents: h.snapshotAgents()}) +} + +func (h *Hub) agentWriter(ac *agentConn) { + ticker := time.NewTicker(agentHeartbeatEvery) + defer ticker.Stop() + for { + select { + case msg, ok := <-ac.send: + if !ok { + return + } + _ = ac.conn.SetWriteDeadline(time.Now().Add(writeWait)) + if err := ac.conn.WriteMessage(websocket.TextMessage, msg); err != nil { + return + } + case <-ticker.C: + _ = ac.conn.SetWriteDeadline(time.Now().Add(writeWait)) + if err := ac.conn.WriteControl(websocket.PingMessage, []byte("ping"), time.Now().Add(writeWait)); err != nil { + return + } + } + } +} + +func (h *Hub) queue(ac *agentConn, msg outbound) { + b, err := json.Marshal(msg) + if err != nil { + return + } + select { + case ac.send <- b: + default: + } +} + +func (h *Hub) sendJSON(conn *websocket.Conn, msg outbound) { + b, _ := json.Marshal(msg) + _ = conn.SetWriteDeadline(time.Now().Add(writeWait)) + _ = conn.WriteMessage(websocket.TextMessage, b) +} + +func (h *Hub) broadcastUI(msg outbound) { + b, err := json.Marshal(msg) + if err != nil { + return + } + h.mu.Lock() + defer h.mu.Unlock() + for conn := range h.uis { + _ = conn.SetWriteDeadline(time.Now().Add(writeWait)) + if err := conn.WriteMessage(websocket.TextMessage, b); err != nil { + _ = conn.Close() + delete(h.uis, conn) + } + } +} + +func (h *Hub) snapshotAgents() []any { + h.mu.Lock() + defer h.mu.Unlock() + out := make([]any, 0, len(h.agents)) + now := time.Now().UTC() + for id, a := range h.agents { + out = append(out, map[string]any{ + "agent_id": id, + "last_seen_s": now.Sub(a.lastSeen).Seconds(), + "connected": true, + }) + } + return out +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + _ = enc.Encode(v) +} diff --git a/apps/streamd/servecmd/servecmd.go b/apps/streamd/servecmd/servecmd.go new file mode 100644 index 0000000..51dfc61 --- /dev/null +++ b/apps/streamd/servecmd/servecmd.go @@ -0,0 +1,245 @@ +// Package servecmd is the streamd control plane, exposed as a library so the +// single drm binary can host it as a subcommand. +package servecmd + +import ( + "flag" + "fmt" + "log" + "net/http" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "syscall" + "time" + + "drmdecryption/apps/streamd/internal/api" + "drmdecryption/apps/streamd/internal/db" + "drmdecryption/apps/streamd/internal/supervisor" + "drmdecryption/apps/streamd/internal/ui" + "drmdecryption/apps/streamd/internal/ws" +) + +// Run dispatches a streamd subcommand. args[0] is the subcommand name. +func Run(args []string) error { + if len(args) < 1 { + Usage() + return fmt.Errorf("streamd: subcommand required") + } + sub, rest := args[0], args[1:] + switch sub { + case "serve": + serveCmd(rest) + case "worker": + return fmt.Errorf("worker: media is supervised in-process by `serve` now") + case "help", "-h", "--help": + Usage() + default: + Usage() + return fmt.Errorf("streamd: unknown subcommand %q", sub) + } + return nil +} + +// Usage prints the streamd subcommand help. +func Usage() { + fmt.Fprintf(os.Stderr, `streamd — Go control plane for DRM restreams + +Usage: + streamd serve [flags] + +serve flags: + --bind listen address (default 0.0.0.0:8083) + --data data dir for sqlite + www + work (default .cache/streamd) + --token shared bearer token for mutating API (or STREAMD_TOKEN) + --nre path to N_m3u8DL-RE (else NRE_PATH, bin/, PATH) + --ffmpeg path to ffmpeg (else FFMPEG_PATH, bin/, PATH) + --mp4decrypt path to mp4decrypt (else MP4DECRYPT_PATH, bin/, PATH) +`) +} + +func serveCmd(args []string) { + fs := flag.NewFlagSet("serve", flag.ExitOnError) + bind := fs.String("bind", "0.0.0.0:8083", "listen address") + data := fs.String("data", "", "data directory (sqlite, www, work, logs)") + token := fs.String("token", "", "API bearer token (default: STREAMD_TOKEN env)") + nre := fs.String("nre", "", "N_m3u8DL-RE binary") + ffmpeg := fs.String("ffmpeg", "", "ffmpeg binary") + mp4decrypt := fs.String("mp4decrypt", "", "mp4decrypt binary") + _ = fs.Parse(args) + + dataDir := *data + if dataDir == "" { + dataDir = filepath.Join(".", ".cache", "streamd") + } + tok := *token + if tok == "" { + tok = os.Getenv("STREAMD_TOKEN") + } + + for _, sub := range []string{"www", "work", "logs"} { + if err := os.MkdirAll(filepath.Join(dataDir, sub), 0o755); err != nil { + log.Fatal(err) + } + } + + store, err := db.Open(dataDir) + if err != nil { + log.Fatalf("db: %v", err) + } + defer store.Close() + + nrePath := firstExisting(*nre, + os.Getenv("NRE_PATH"), + os.Getenv("N_M3U8DL_RE"), + filepath.Join("bin", "N_m3u8DL-RE.exe"), + "N_m3u8DL-RE", + ) + ffPath := firstExisting(*ffmpeg, + os.Getenv("FFMPEG_PATH"), + os.Getenv("FFMPEG"), + filepath.Join("bin", "ffmpeg.exe"), + "ffmpeg", + ) + decPath := firstExisting(*mp4decrypt, + os.Getenv("MP4DECRYPT_PATH"), + os.Getenv("MP4DECRYPT"), + filepath.Join("bin", "mp4decrypt.exe"), + "mp4decrypt", + ) + + sup := supervisor.NewMedia(supervisor.Options{ + Store: store, + DataDir: dataDir, + NRE: nrePath, + FFmpeg: ffPath, + MP4Decrypt: decPath, + Log: log.Default(), + }) + defer sup.Close() + + apiH := &api.Handler{ + Store: store, + Token: tok, + Supervisor: sup, + StartedAt: time.Now(), + } + hub := ws.NewHub(store, tok, log.Default()) + + mux := http.NewServeMux() + apiH.Mount(mux) + hub.Mount(mux) + mux.Handle("/", ui.Handler()) + mux.Handle("/static/", http.StripPrefix("/static/", ui.Static())) + mux.Handle("/hls/", http.StripPrefix("/hls/", http.FileServer(http.Dir(filepath.Join(dataDir, "www"))))) + + srv := &http.Server{ + Addr: *bind, + Handler: withCORS(mux), + ReadHeaderTimeout: 10 * time.Second, + } + + log.Printf("streamd listening on http://%s data=%s", *bind, dataDir) + log.Printf("media: nre=%s", nrePath) + log.Printf("media: ffmpeg=%s", ffPath) + log.Printf("media: mp4decrypt=%s", decPath) + log.Printf("ws: agents=/ws/agent ui=/ws/ui") + if tok == "" { + log.Printf("warning: no --token / STREAMD_TOKEN set; mutating API is open") + } + + stopBG := make(chan struct{}) + go hub.RunExpireLoop(stopBG) + + // Resume enabled streams that already have credentials (staggered). + go func() { + time.Sleep(500 * time.Millisecond) + list, err := store.ListStreams() + if err != nil { + return + } + for _, st := range list { + if !st.Enabled || strings.TrimSpace(st.MPD) == "" || strings.TrimSpace(st.Key) == "" { + continue + } + mpd := strings.ToLower(strings.TrimSpace(st.MPD)) + if !strings.HasPrefix(mpd, "http://") && !strings.HasPrefix(mpd, "https://") { + log.Printf("auto-start skip %s: mpd is not an http(s) URL", st.Name) + continue + } + log.Printf("auto-start enabled stream %s", st.Name) + func(id int64, name string) { + defer func() { + if r := recover(); r != nil { + log.Printf("auto-start %s panic: %v", name, r) + } + }() + if err := sup.Start(id); err != nil { + log.Printf("auto-start %s: %v", name, err) + } + }(st.ID, st.Name) + time.Sleep(1500 * time.Millisecond) + } + }() + + go func() { + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt, syscall.SIGTERM) + <-ch + log.Printf("shutting down…") + close(stopBG) + sup.Close() + _ = srv.Close() + }() + + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatal(err) + } +} + +func withCORS(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, X-Streamd-Token") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS") + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusNoContent) + return + } + next.ServeHTTP(w, r) + }) +} + +func firstExisting(explicit string, candidates ...string) string { + if explicit != "" { + if st, err := os.Stat(explicit); err == nil && !st.IsDir() { + return explicit + } + // still return explicit so exec fails clearly if user set a bad path + if filepath.IsAbs(explicit) || containsSep(explicit) { + return explicit + } + } + for _, c := range candidates { + if c == "" { + continue + } + if st, err := os.Stat(c); err == nil && !st.IsDir() { + return c + } + if p, err := execLookPath(c); err == nil { + return p + } + } + return explicit +} + +func containsSep(s string) bool { + return filepath.Base(s) != s +} + +func execLookPath(file string) (string, error) { + return exec.LookPath(file) +} diff --git a/apps/wvkey/README.md b/apps/wvkey/README.md new file mode 100644 index 0000000..b9dee9c --- /dev/null +++ b/apps/wvkey/README.md @@ -0,0 +1,60 @@ +# wvkey — Widevine CDM helper + +The only Python in the project. Go (`apps/pkg/wvkey`) shells out to it to turn a +PSSH plus a license endpoint into `KID:KEY` lines via pywidevine. + +```text +apps/wvkey/ + wvkey.py + requirements.txt +``` + +Keep it thin: it does the CDM exchange and nothing else. + +## Setup + +```bash +# from the repo root +python -m venv .venv +.venv/bin/pip install -r apps/wvkey/requirements.txt # Windows: .venv\Scripts\pip +``` + +Put your Widevine device file somewhere gitignored, e.g. `data/device.wvd`. The Go +side finds `.venv` automatically; override with `--python`. + +Every unique value is a flag. There are no baked-in device paths, account URLs or +user agents. + +## Run + +Two license shapes: + +```bash +# JSON challenge wrapper with an Authorization header +.venv/bin/python apps/wvkey/wvkey.py \ + --mode modulardrm \ + --wvd data/device.wvd \ + --pssh '' \ + --auth 'Bearer ...' \ + --pid '' \ + --license-url 'https://...' \ + --quiet + +# raw binary challenge (octet-stream) +.venv/bin/python apps/wvkey/wvkey.py \ + --mode raw \ + --wvd data/device.wvd \ + --pssh '' \ + --license-url 'https://...' \ + --quiet +``` + +Options: `--user-agent`, `--all` (print every CONTENT key — needed for multi-KID +streams). + +Which mode an app needs is declared by its module (`KeyMode()`), never sniffed from +the license URL at runtime. See [docs/capture.md](../../docs/capture.md) for how to +tell them apart from a capture. + +Go callers use `wvkey.Fetch` / `FetchRaw` / `FetchRawAll` and always pass `Script`, +`WVD` and `LicenseURL`. diff --git a/apps/wvkey/requirements.txt b/apps/wvkey/requirements.txt new file mode 100644 index 0000000..9c8759c --- /dev/null +++ b/apps/wvkey/requirements.txt @@ -0,0 +1,6 @@ +# Dependencies for wvkey.py (the only Python in the project). +# Install into a venv at the repo root: +# python -m venv .venv +# .venv/bin/pip install -r apps/wvkey/requirements.txt +pywidevine==1.9.0 +requests==2.34.2 diff --git a/apps/wvkey/wvkey.py b/apps/wvkey/wvkey.py new file mode 100644 index 0000000..88fd19e --- /dev/null +++ b/apps/wvkey/wvkey.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Thin Widevine CDM helper for the Go capture tooling. + +Prints a single CONTENT key as KID:KEY on stdout (no banners) when --quiet. +Exit code 0 on success. + +Example: + .venv/Scripts/python.exe wvkey.py \\ + --wvd path/to/device.wvd \\ + --pssh 'AAAA…' \\ + --license-url 'https://…' \\ + --auth 'Basic …' \\ + --pid 'xxxx' \\ + --quiet +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import sys +from pathlib import Path + +import requests +from pywidevine.cdm import Cdm +from pywidevine.device import Device +from pywidevine.pssh import PSSH + + +def fetch_content_keys_modulardrm( + *, + wvd: Path, + pssh_b64: str, + auth: str, + release_pid: str, + license_url: str, + user_agent: str, +) -> list[str]: + """RTE / thePlatform ModularDrm (JSON challenge wrapper).""" + device = Device.load(str(wvd)) + cdm = Cdm.from_device(device) + session_id = cdm.open() + try: + challenge = cdm.get_license_challenge(session_id, PSSH(pssh_b64)) + challenge_b64 = base64.b64encode(challenge).decode("utf-8") + payload = { + "getWidevineLicense": { + "releasePid": release_pid, + "widevineChallenge": challenge_b64, + } + } + headers = { + "Accept-Encoding": "gzip", + "Authorization": auth, + "Content-Type": "application/json; charset=utf-8", + } + if user_agent: + headers["User-Agent"] = user_agent + response = requests.post(license_url, json=payload, headers=headers, timeout=30) + data = response.json() + if data.get("isException"): + raise RuntimeError(data.get("description") or "license server exception") + license_b64 = ( + data.get("getWidevineLicenseResponse", {}).get("license") + or data.get("license") + ) + if not license_b64: + raise RuntimeError("no license in response: " + json.dumps(data)[:1500]) + cdm.parse_license(session_id, base64.b64decode(license_b64)) + keys = [ + f"{key.kid.hex}:{key.key.hex()}" + for key in cdm.get_keys(session_id) + if key.type == "CONTENT" + ] + finally: + cdm.close(session_id) + if not keys: + raise RuntimeError("no CONTENT keys") + return keys + + +def fetch_content_keys_raw( + *, + wvd: Path, + pssh_b64: str, + license_url: str, + user_agent: str, +) -> list[str]: + """Brightcove / generic Widevine: POST raw challenge, parse raw license body.""" + device = Device.load(str(wvd)) + cdm = Cdm.from_device(device) + session_id = cdm.open() + try: + challenge = cdm.get_license_challenge(session_id, PSSH(pssh_b64)) + headers = { + "Content-Type": "application/octet-stream", + "Accept": "*/*", + } + if user_agent: + headers["User-Agent"] = user_agent + response = requests.post( + license_url, data=challenge, headers=headers, timeout=30 + ) + if response.status_code >= 400: + raise RuntimeError( + f"license HTTP {response.status_code}: {response.text[:500]!r}" + ) + body = response.content + if not body: + raise RuntimeError("empty license response body") + # Some servers wrap base64 text; try raw first, then b64. + try: + cdm.parse_license(session_id, body) + except Exception: + try: + cdm.parse_license(session_id, base64.b64decode(body)) + except Exception as exc: + raise RuntimeError( + f"parse_license failed ({len(body)} bytes): {exc}" + ) from exc + keys = [ + f"{key.kid.hex}:{key.key.hex()}" + for key in cdm.get_keys(session_id) + if key.type == "CONTENT" + ] + finally: + cdm.close(session_id) + if not keys: + raise RuntimeError("no CONTENT keys") + return keys + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description="Widevine CDM helper → KID:KEY") + p.add_argument("--wvd", type=Path, required=True, help="Path to .wvd device file") + p.add_argument("--pssh", required=True, help="Base64 PSSH box") + p.add_argument( + "--mode", + choices=("modulardrm", "raw"), + default="modulardrm", + help="modulardrm=RTE JSON; raw=Brightcove/octet-stream challenge", + ) + p.add_argument("--auth", default="", help="Authorization header (modulardrm)") + p.add_argument("--pid", default="", help="releasePid (modulardrm)") + p.add_argument( + "--license-url", + required=True, + help="Widevine license server URL", + ) + p.add_argument( + "--user-agent", + default="", + help="Optional User-Agent for the license request", + ) + p.add_argument( + "--quiet", + action="store_true", + help="Print only KID:KEY lines (for Go tooling)", + ) + p.add_argument( + "--all", + action="store_true", + help="Print every CONTENT key (default: first only)", + ) + args = p.parse_args(argv) + + if not args.wvd.is_file(): + print(f"[!] missing WVD: {args.wvd}", file=sys.stderr) + return 2 + + try: + if args.mode == "raw": + keys = fetch_content_keys_raw( + wvd=args.wvd, + pssh_b64=args.pssh, + license_url=args.license_url, + user_agent=args.user_agent, + ) + else: + if not args.auth or not args.pid: + print("[!] --auth and --pid required for modulardrm mode", file=sys.stderr) + return 2 + keys = fetch_content_keys_modulardrm( + wvd=args.wvd, + pssh_b64=args.pssh, + auth=args.auth, + release_pid=args.pid, + license_url=args.license_url, + user_agent=args.user_agent, + ) + except Exception as exc: + print(f"[!] {exc}", file=sys.stderr) + return 1 + + out = keys if args.all else keys[:1] + if not args.quiet: + print("[+] Keys:", file=sys.stderr) + for pair in out: + print(f"--key {pair}", file=sys.stderr) + for pair in out: + print(pair) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..f4ddeb0 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,40 @@ +# Documentation + +| Doc | Read it when | +|---|---| +| [quickstart.md](quickstart.md) | You just cloned the repo and want something working | +| [architecture.md](architecture.md) | You want to know how the pieces fit and why | +| [modules.md](modules.md) | You are adding support for a new streaming app | +| [capture.md](capture.md) | You need to discover an app's URLs, IDs and UI selectors | +| [streamd.md](streamd.md) | You are running the control plane + agent | +| [providers/rte.md](providers/rte.md) | You are filling in `apps/modules/rte/module.yaml` | +| [providers/tg4.md](providers/tg4.md) | You are filling in `apps/modules/tg4/module.yaml` | + +## The one idea worth knowing first + +The repo separates **logic** from **values**: + +```text +apps/modules/tg4/*.go tracked. How to drive the app. No secrets, ever. +apps/modules/tg4/module.yaml gitignored. Account IDs, policy keys, video IDs, KIDs. +``` + +Both module packages have a test asserting an empty config yields empty +credentials, so the separation is checked rather than merely intended: + +```bash +go -C apps/modules test ./... -run NoHardcoded -v +``` + +A fresh clone therefore builds and runs, but knows no channels until you supply +your own values. Each `providers/*.md` doc explains where to get them. + +## Build + +```bash +go -C apps/cli build -o ../../bin/drm . # Windows: -o ../../bin/drm.exe +./bin/drm proxy build # once, for phone capture +``` + +There is no build script and no Makefile — `go build` is the build system. +[quickstart.md](quickstart.md) lists every command. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..8dfbf2a --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,241 @@ +# Architecture + +## What the system does + +Given an Android app that streams DRM-protected live TV, it recovers the three +things a downloader needs — **manifest URL**, **Widevine content keys**, and any +**auth headers** — and keeps them fresh. + +Two independent ways to get them: + +1. **Catalog** (`drm catalog`) — the app's playback catalog is public, so a plain + HTTPS request plus a local CDM is enough. No phone. +2. **Phone capture** (`drm capture`) — launch the real app under an on-device + HTTPS MITM, drive its UI to the channel, and read the license exchange. + +What it deliberately does not do: break DRM. The CDM used is a provisioned +Widevine device you supply (`.wvd`); the system only automates the request flow a +real client performs. + +## One binary + +```text + ┌──────────────────────── bin/drm ────────────────────────┐ + │ │ + modules ───┤ capture catalog agent serve proxy │ + linked in │ │ + └─────────────────────────────────────────────────────────┘ +``` + +`apps/cli` is the only `main`. It blank-imports `drmdecryption/modules/all`, which +is what makes `--app ` resolve. Each app module registers itself: + +```go +func init() { + appreg.Register(Name, New) // --app rte + appreg.RegisterFlags(bindFlags) // --rte. overrides +} +``` + +Adding a provider means adding a package and one import line. There is no plugin +loading, no scripting runtime, and nothing to deploy beside the binary. + +## Logic vs values + +This is the central constraint: + +| `apps/modules//*.go` — **tracked** | `apps/modules//module.yaml` — **gitignored** (bbc published) | +|---|---| +| launch + auto-play sequence | android package id | +| navigation idioms (chip→Play, drawer→Live) | license URL, origin hostnames | +| manifest rewrite shape | channel KIDs, account id, policy key | +| catalog request flow | video ids, playback config URL | +| which capture fields are required | UI selector regexes, labels, aliases | + +Module Go source contains no account id, policy key, video id, license URL, origin +host or KID. Values arrive by **flag → environment → module.yaml → Go default**, +and only mechanical things (timeouts, DASH timescales, card geometry) have +defaults in code. + +`.gitignore` enforces the split: + +```gitignore +apps/modules/** +!apps/modules/README.md +!apps/modules/go.mod +!apps/modules/go.sum +!apps/modules/*/ +!apps/modules/*/*.go +!apps/modules/*/**/*.go +``` + +The directory re-include is required — git will not descend into an excluded +directory to find negated files. + +## Go modules + +Six modules, wired with `replace`: + +```text +apps/pkg drmdecryption shared, provider-neutral +apps/modules drmdecryption/modules app modules -> pkg +apps/capture drmdecryption/apps/capture capture command -> pkg +apps/agent drmdecryption/apps/agent agent -> pkg +apps/streamd drmdecryption/apps/streamd control plane -> pkg +apps/proxy drmdecryption/apps/proxy MITM host CLI -> pkg +apps/cli drmdecryption/apps/cli the binary -> all of the above + +apps/proxy/device appproxy standalone, cross-compiled for the phone +``` + +Command bodies live in non-`internal` packages (`capturecmd`, `agentcmd`, +`servecmd`, `proxyctlcmd`) because `internal/` is not importable across module +boundaries and `apps/cli` has to call them. + +## Plugin contract + +`apps/pkg/app` defines what a module must provide and what it may add. + +Required (`app.App`): + +| Method | Purpose | +|---|---| +| `Name`, `Package`, `LicenseURL` | identity | +| `Launch(c)` | cold-start the app, wait for its home UI | +| `AutoPlay(c, channel)` | navigate to a channel and confirm playback | +| `CaptureHints()` | which capture fields are required, plus URL scoring | +| `KeyMode()` | `"modulardrm"` or `"raw"` — never sniffed from the license URL | +| `MPDRewriter()` | manifest rewrite, or `mpd.Passthrough{}` | +| `Channels`, `HasChannel` | channel list and lookup | +| `ProxyBin`, `CAHash` | on-device MITM binary and CA hash | + +Optional, detected by type assertion: + +| Interface | Gives you | +|---|---| +| `app.Catalog` | `drm catalog --app ` works | +| `app.CatalogLister` | a richer `catalog --list` with catalog ids and provider flags | +| `app.StreamDefaults` | streamd downloader settings (selectors, rewriter, buffering) | +| `mpd.Rewriter` + `mpd.Register` | a named manifest rewrite streamd can resolve | + +## Shared libraries (`apps/pkg`) + +| Package | Role | +|---|---| +| `app` | plugin interface + registry + optional capability interfaces | +| `adb` | ADB client: shell, push/pull, UI dump, tap, swipe, playback probe | +| `uiflow` | UI primitives modules drive playback with — every threshold a parameter | +| `capture` | parse on-device capture JSON; score candidate manifest URLs | +| `phonecap` | one capture job end to end | +| `proxy` | push/start/stop the on-device MITM, CA install/reinject, pull captures | +| `wvkey` | shells out to `apps/wvkey/wvkey.py` | +| `brightcove` | Brightcove Playback API + HLS Widevine extraction — a *platform*, not a provider | +| `mpd` | rewriter interface + registry, synthetic live manifest builder, local server | +| `modcfg` | load a module's `module.yaml`, apply env overrides, resolve paths | +| `session` | write `outputs///` | +| `repo` | locate the repo root | + +Nothing here names a streaming provider. + +## Phone capture pipeline + +```text +drm capture --app X --channel Y --auto-play + │ + ├─ push + start apps/proxy/device (appproxy) on the phone + ├─ set the phone's global http_proxy to 127.0.0.1:8080 + ├─ Magisk-mount the MITM CA into the system trust store + ├─ module.Launch() : force-stop → monkey launch → wait for a home marker + ├─ module.AutoPlay() : app-specific navigation → wait for playback + ├─ capture.Wait() : poll the device capture JSON + mirrored proxy log + │ until the module's required fields appear + ├─ wvkey.py : PSSH + license → KID:KEY (mode from module.KeyMode()) + └─ session.Write() : outputs/// + latest/ +``` + +The MITM runs **on the device**, so it cannot read a module's values file. A +module's manifest-scoring hosts are passed to it as flags +(`capture.ScoreCfg.ProxyArgs()` → `-score-host`, `-score-deny`). + +### Why URL scoring exists + +A MITM sees EPG feeds, analytics beacons and config blobs alongside the real +manifest. `capture.ScoreCfg` ranks candidates: format-level signals (`.mpd`, +`.m3u8`, `playlist-hls`, `/manifest`, `.isml`) are generic and live in `apps/pkg`; +provider hosts and noise needles come from `module.yaml`. + +## Manifest rewriting + +Some providers serve an ad-stitched manifest whose session expires (HTTP 410) +while the underlying origin keeps running. `apps/pkg/mpd` can publish a +**synthetic** manifest on the origin encoder's own segment grid so a downloader +keeps fetching real segments: + +```text +mpd.LearnGrid fetch the origin Smooth manifest once; derive segment + duration and phase offset per media type +mpd.BuildSynthetic emit a dynamic MPD on that grid, with a sliding window +mpd.StartLocal serve it on 127.0.0.1 and re-run the rewrite on every GET +mpd.Register/Lookup resolve a rewriter by name, so streamd needs no provider import +``` + +`SyntheticConfig` holds every value that could differ — timescales, stream names, +bandwidths, codecs, resolution, segment templates, live-edge offset, window size. +The RTE module supplies origins and KIDs; when the upstream manifest is gone, a +`mpd.KIDResolver` maps the KID from a previous capture back to its origin. + +## Control plane (`drm serve`) + +| Piece | Path | +|---|---| +| Command | `apps/streamd/servecmd` | +| HTTP API | `apps/streamd/internal/api` | +| SQLite | `apps/streamd/internal/db` | +| Dashboard | `apps/streamd/internal/ui` | +| Media supervisor | `apps/streamd/internal/supervisor` | + +Port 8083 by default. Mutating routes need `Authorization: Bearer `. + +Streamd stores no provider knowledge. `GET /api/apps` reports the compiled-in +modules, and the supervisor resolves per-stream downloader settings at start time: + +```text +settingsFor(stream): + stored row -> the stream's app module (app.StreamDefaults) -> neutral defaults +``` + +The rewriter is a registry lookup on the stream's `rewriter` column, and HLS is +detected from the manifest shape — not from an app name or a stream name. + +## Agent (`drm agent run`) + +```text +poll streamd ──► durable SQLite queue ──► claim a free ADB device + │ + POST credentials ◄── phone capture (phonecap) +``` + +Per-stream claims expire via a websocket heartbeat, so a dead agent releases its +phones. The agent never installs apps and never runs a downloader. It drives +whichever module a stream names; it contains no provider logic. + +## Where things land + +| Path | Contents | +|---|---| +| `data/` | durable secrets: `.wvd`, CA — gitignored | +| `outputs///` | capture + catalog sessions, with `latest/` as a copy | +| `outputs/discover//` | MITM discover dumps | +| `.cache/streamd/` | streamd SQLite, work dirs, HLS output | +| `.cache/ui-/` | UI dumps used by the automation | + +## Honest status + +| Area | State | +|---|---| +| Capture (catalog + phone), both modules | working, verified against live services | +| Compiled-in modules, one binary | done | +| streamd API + SQLite + dashboard + claims | done | +| Synthetic manifest rewrite | done, verified against a live origin | +| Agent queue + phone refresh | done | +| Real downloader/ffmpeg workers in streamd | partial — the supervisor shells out to external tools and is the least exercised path | diff --git a/docs/capture.md b/docs/capture.md new file mode 100644 index 0000000..992d6dd --- /dev/null +++ b/docs/capture.md @@ -0,0 +1,262 @@ +# Using the capture binary + +`drm capture` runs an HTTPS MITM **on the phone**, drives the app, and records the +DRM exchange. It is both the production capture path and the tool you use to +discover everything a new module needs. + +## Device setup (once) + +```bash +adb devices # phone listed and authorised +./bin/drm proxy build # cross-compile the on-device MITM (linux/arm64) +./bin/drm proxy install-ca --reinject +``` + +The phone must be rooted (Magisk). `install-ca` pushes the MITM CA and bind-mounts +it over the system trust store, including into already-running zygote namespaces — +a user-store CA is not enough, modern apps ignore it. + +Verify the mount: + +```bash +adb shell 'ls /system/etc/security/cacerts/ | grep 6c3578b4' +adb shell 'su -c "ls /apex/com.android.conscrypt/cacerts/6c3578b4.0"' +``` + +If an app still reports `tls: unknown certificate`, re-run the reinject and +force-stop the app so it inherits the mount: + +```bash +./bin/drm proxy reinject-ca +adb shell am force-stop +``` + +Keep the screen on for the whole session — UI automation cannot tap a dark screen: + +```bash +adb shell svc power stayon true +adb shell settings put system screen_off_timeout 1800000 +``` + +Restore the timeout afterwards (default is usually 60000). + +## Three modes + +### Passive — no module needed + +```bash +./bin/drm capture --wait 300 +``` + +Starts the MITM and nothing else. You open any app and play something; it records +whatever license, PSSH and manifest traffic appears and writes +`outputs/capture//`. This is the first thing to run against an unknown app. + +### Launch only + +```bash +./bin/drm capture --app myapp --wvd data/device.wvd +``` + +Launches the app, then waits while you navigate by hand. Useful while you are still +working out the UI sequence. + +### Full automation + +```bash +./bin/drm capture --app myapp --channel main --auto-play --wvd data/device.wvd +``` + +Runs the module's `Launch` and `AutoPlay`, then waits for the required fields. + +## Flags + +| Flag | Default | Purpose | +|---|---|---| +| `--app` | _(empty)_ | module to drive; empty means passive | +| `--channel` | — | channel id, required by `--auto-play` | +| `--auto-play` | false | run the module's navigation | +| `--wait` | 180 | seconds to wait for the capture to complete | +| `--wvd` | — | Widevine device file; required to fetch keys | +| `--python` | `.venv` | interpreter for `wvkey.py` | +| `--key-mode` | from the module | force `modulardrm` or `raw` | +| `--user-agent` | — | UA for the license request | +| `--serial` | sole device | target a specific phone | +| `--close` | true | force-stop the app when done | +| `--.` | — | override any of that module's values | + +## What a run looks like + +```text +[*] Stopping any old appproxy... +[*] Pushing appproxy... +[+] appproxy pid=7604; capture -> /data/local/tmp/appproxy_cap.json +[*] Setting HTTP proxy -> 127.0.0.1:8080 (was "null") +[*] Re-injecting system CA (Magisk)... +[+] System CA present in conscrypt +[*] Launching rte (air.RTE.OSMF.Minimal)… +[*] Auto-play RTE 2… +[*] tap_ui "Live"/"Live tab, 2 out of 5" @ 324,2253 +[*] tap chip @ 356,297 +[*] tap Play @ 540,720 +[+] air.RTE.OSMF.Minimal is PLAYING +[*] Waiting for license + PSSH + manifest… +[+] Capture: + pid: RKRPCEib8z9V + mpd: https://dai.google.com/linear/dash/pa/event/.../stream/... + license: https://widevine.entitlement.eu.theplatform.com/wv/web/ModularDrm?... + pssh: AAAASnBzc2gAAAAA7e+LqXnWSs6jyCfc1R0h7QAAACoiIGRmMTYzMzgyMWRk… +[+] key: 5490781a77572107ec251237e94e3b53:cee5fc45b70040ffea2b2576fb5abda5 +[+] session: outputs/rte/20261005-220252/session.json +[*] Closing air.RTE.OSMF.Minimal… +[*] HTTP proxy cleared +``` + +### Output + +```text +outputs/// + session.json everything, machine-readable + mpd.txt manifest URL + key.txt KID:KEY (one per line for multi-key streams) + pssh.txt Widevine init data + auth.txt Authorization value, when the app uses one + pid.txt provider id, when the app uses one +outputs//latest/ a copy of the most recent run +``` + +Live log while a capture runs: `.cache/appproxy-.log`. + +--- + +## Discovering a new app + +### 1. Record everything + +```bash +./bin/drm proxy discover --install-ca --reinject +# play the app on the phone; Ctrl+C when done +``` + +Writes `outputs/discover//`: + +| File | Contents | +|---|---| +| `appproxy_traffic.jsonl` | every HTTP(S) request/response, one JSON object per line | +| `appproxy_cap.json` | the structured fields the proxy recognised | +| `appproxy.log` | tagged lines: `[MPD]`, `[LIC]`, `[PSSH]`, `[MAN]` | + +### 2. Mine it + +```bash +# the android package id +adb shell dumpsys window | grep mCurrentFocus + +# license endpoint +grep -iE 'license|licence|widevine|/lic/' outputs/discover/*/appproxy.log + +# manifest candidates +grep -oE 'https?://[^"]+\.(m3u8|mpd)' outputs/discover/*/appproxy_traffic.jsonl \ + | sort -u + +# the hosts involved, by frequency +grep -oE '"host":"[^"]+"' outputs/discover/*/appproxy_traffic.jsonl \ + | sort | uniq -c | sort -rn | head -20 + +# config blobs that often carry ids and tokens +grep -oE 'https?://[^"]+(config|playback)[^"]*\.json' \ + outputs/discover/*/appproxy_traffic.jsonl | sort -u +``` + +What to pull out, and where it goes in `module.yaml`: + +| Found | Key | +|---|---| +| package id | `package` | +| license endpoint | `license_url` | +| CDN host serving the manifest | `score.hosts` | +| analytics / EPG hosts polluting the results | `score.deny` | +| account ids, policy keys, video ids | provider-specific keys | + +### 3. Work out the UI + +Dump the tree at each step and read off stable selectors: + +```bash +adb shell uiautomator dump /sdcard/ui.xml +adb shell cat /sdcard/ui.xml > ui.xml +``` + +Extract the useful attributes: + +```bash +grep -oE 'resource-id="[^"]*"' ui.xml | sort -u +grep -oE 'content-desc="[^"]*"' ui.xml | sort -u +``` + +Prefer `resource-id` over `content-desc` or `text` — ids survive translation and +copy changes. Check a candidate marker really exists before you rely on it; a +launch marker that never matches costs the full timeout on every capture and is +easy to miss because the pipeline recovers anyway. + +Test taps by hand before writing Go: + +```bash +adb shell input tap 540 732 +adb shell input swipe 540 1700 540 700 350 +``` + +### 4. Determine the key mode + +| The app sends | Use | +|---|---| +| a JSON wrapper around the challenge, with an Authorization header | `modulardrm` | +| a raw binary challenge, `Content-Type: application/octet-stream` | `raw` | + +Check the license request body in `appproxy_traffic.jsonl`. Then set `KeyMode()` +in the module — never sniff it from the URL at runtime. + +### 5. Decide which fields are required + +Look at what `appproxy_cap.json` actually contained after a successful play: + +- DASH + JSON license → usually `auth`, `pid`, `pssh`, `mpd` +- HLS + binary license → usually `license_url`, `mpd` (the PSSH comes from the + playlist afterwards) + +Pass exactly those to `a.Hints(...)`. Asking for a field the app never emits makes +every capture time out. + +### 6. Verify + +Where an app has a public catalog, capture the same channel both ways and compare — +independent paths agreeing on the PSSH and key set is strong evidence both are +right: + +```bash +./bin/drm catalog --app myapp --channel main --keys --wvd data/device.wvd --json > a.json +./bin/drm capture --app myapp --channel main --auto-play --wvd data/device.wvd +diff <(jq -r '.keys[]' a.json | sort) <(sort outputs/myapp/latest/key.txt) +``` + +Also sanity-check that the KID you captured matches whatever you recorded in +`module.yaml` for that channel. + +## Troubleshooting + +| Symptom | Likely cause | +|---|---| +| `tls: unknown certificate` in the app | CA not mounted into conscrypt; reinject, then force-stop the app | +| `timed out waiting for capture ([auth pid pssh mpd])` | playback never started, or `Hints` asks for a field this app does not send | +| `launch: ui node not found within Ns` | launch marker does not match — dump the tree and pick a resource id | +| `launch: exit status 1` | monkey raced a force-stop; usually transient, retry | +| `auto-play failed: ... not found` | UI changed, or the phone is on a different screen than expected | +| manifest field holds an EPG or config URL | add that host to `score.deny` | +| `CA inject skipped/failed` | transient after repeated runs; re-run `proxy reinject-ca` | +| Screen asleep mid-run | `adb shell svc power stayon true` | + +Clear the phone's proxy if a run is interrupted: + +```bash +./bin/drm proxy clear-proxy +``` diff --git a/docs/modules.md b/docs/modules.md new file mode 100644 index 0000000..db637ed --- /dev/null +++ b/docs/modules.md @@ -0,0 +1,379 @@ +# Writing an app module + +A module teaches the binary how to drive one streaming app. It is a Go package, it +is compiled in, and it contains no secrets. + +Work through [capture.md](capture.md) first — you need the app's package id, +license URL, manifest host and UI selectors before you can write any of this. + +## Anatomy + +```text +apps/modules/myapp/ + config.go Config struct + defaults + flag binding + myapp.go init(), the App type, Launch, AutoPlay, capability methods + steps.go navigation idioms specific to this app + catalog.go optional: resolve channels without a phone + mpd.go optional: manifest rewriting + module.yaml GITIGNORED - your values +``` + +Everything is registered from `init()`, and one line in `apps/modules/all/all.go` +links it into the binary. + +## 1. Config + +Embed `provider.Common` for the fields every module has, then add your own. + +```go +package myapp + +import ( + "drmdecryption/modcfg" + "drmdecryption/modules/provider" +) + +const Name = "myapp" + +type Channel struct { + Label string `yaml:"label"` + LiveCardIndex int `yaml:"live_card_index"` + PlayDesc []string `yaml:"play_desc"` +} + +type Config struct { + provider.Common `yaml:",inline"` + + // Provider values. Never give these defaults in code. + APIBase string `yaml:"api_base"` + Channels map[string]Channel `yaml:"channels"` + + Timeouts Timeouts `yaml:"timeouts"` + UI UI `yaml:"ui"` +} + +type Timeouts struct { + // provider.Duration accepts "25s" and bare seconds; time.Duration alone + // would force raw nanoseconds in YAML. + LaunchWaitUI provider.Duration `yaml:"launch_wait_ui"` + Playback provider.Duration `yaml:"playback"` +} + +type UI struct { + LaunchResourceContains []string `yaml:"launch_resource_contains"` + LaunchDescContains []string `yaml:"launch_desc_contains"` +} + +func (c Config) withDefaults() Config { + t := &c.Timeouts + t.LaunchWaitUI = provider.Duration(t.LaunchWaitUI.D(25 * time.Second)) + t.Playback = provider.Duration(t.Playback.D(45 * time.Second)) + return c +} +``` + +`provider.Common` gives you `package`, `license_url`, `proxy_bin`, `ca_hash`, +`aliases` and `score`. + +### Loading and overrides + +```go +var flags struct { + packageName string + apiBase string +} + +func loadConfig() (Config, modcfg.Result, error) { + var cfg Config + res, err := modcfg.Load(Name, &cfg) // reads apps/modules/myapp/module.yaml + if err != nil { + return cfg, res, err + } + cfg.Package = modcfg.Override(Name, "package", flags.packageName, cfg.Package) + cfg.APIBase = modcfg.Override(Name, "api_base", flags.apiBase, cfg.APIBase) + return cfg.withDefaults(), res, nil +} +``` + +`modcfg.Override` applies **flag → environment (`MYAPP_API_BASE`) → file**. A +missing `module.yaml` is not an error: the module must still construct so +`drm modules` can report it as unconfigured. + +## 2. The App type + +```go +func init() { + appreg.Register(Name, New) + appreg.RegisterFlags(bindFlags) +} + +func bindFlags(fs *flag.FlagSet) { + fs.StringVar(&flags.packageName, Name+".package", "", "override "+Name+" package id") + fs.StringVar(&flags.apiBase, Name+".api-base", "", "override "+Name+" API base") +} + +type App struct { + *provider.Base + cfg Config +} + +func New() (appreg.App, error) { + cfg, res, err := loadConfig() + if err != nil { + return nil, err + } + labels := map[string]string{} + for id, ch := range cfg.Channels { + labels[id] = ch.Label + } + return &App{Base: provider.New(Name, cfg.Common, labels, res), cfg: cfg}, nil +} +``` + +Flag binding is a separate hook from construction on purpose: flags must be bound +to the config holder *before* any app is built, or overrides would be read too +late. + +`provider.Base` supplies `Name`, `Package`, `LicenseURL`, `ProxyBin`, `CAHash`, +`HasChannel`, `Label`, `ChannelIDs`, `CacheDir`, alias-aware `Resolve`, and +`Hints(require…)` preloaded with your `score:` config. + +### Required methods + +```go +func (a *App) KeyMode() string { return "raw" } // or "modulardrm" + +func (a *App) CaptureHints() capture.Hints { + return a.Hints("license_url", "mpd") // fields the MITM must yield +} + +func (a *App) MPDRewriter() mpd.Rewriter { return mpd.Passthrough{} } + +func (a *App) Channels() []appreg.Channel { + out := make([]appreg.Channel, 0, len(a.cfg.Channels)) + for _, id := range a.ChannelIDs() { + out = append(out, appreg.Channel{ID: id, Label: a.Label(id)}) + } + return out +} +``` + +`Hints(...)` names the JSON keys that must be present before a capture counts as +complete. Common sets: + +| Field | Meaning | +|---|---| +| `mpd` | manifest URL (DASH or HLS — the field name is historical) | +| `license_url` | license endpoint seen on the wire | +| `pssh` | Widevine init data | +| `auth` | Authorization header value | +| `pid` | provider-specific id (e.g. a release/program id) | + +Ask only for what the app actually produces. Requiring `pssh` from an HLS app that +puts it in the playlist will hang the capture. + +## 3. Launch and AutoPlay + +```go +func (a *App) Launch(c *adb.Client) error { + c.EnsureAwake() + c.ForceStop(a.Package()) + time.Sleep(400 * time.Millisecond) + if err := uiflow.MonkeyLaunch(c, a.Package()); err != nil { + return err + } + time.Sleep(time.Duration(a.cfg.Timeouts.LaunchSettle)) + c.DismissShadeIfFocused() + return uiflow.WaitUI(c, a.CacheDir(), uiflow.Match{ + ResourceContains: a.cfg.UI.LaunchResourceContains, + DescContains: a.cfg.UI.LaunchDescContains, + }, time.Duration(a.cfg.Timeouts.LaunchWaitUI)) +} + +func (a *App) AutoPlay(c *adb.Client, channel string) error { + id, err := a.Base.Resolve(channel) + if err != nil { + return err + } + ch := a.cfg.Channels[id] + + if err := a.openLivePage(c); err != nil { + return err + } + if err := uiflow.TapCard(c, a.CacheDir(), ch.LiveCardIndex, + time.Duration(a.cfg.Timeouts.Card), a.cfg.Cards); err != nil { + return err + } + return uiflow.WaitPlayback(c, a.CacheDir(), a.Package(), uiflow.PlaybackOpts{ + Timeout: time.Duration(a.cfg.Timeouts.Playback), + SoftFail: true, + OrResourceContains: a.cfg.UI.PlaybackResourceContains, + }) +} +``` + +A `Launch` error is logged and tolerated by the capture pipeline, so pick a launch +marker that genuinely exists — otherwise every run silently burns the full timeout +before `AutoPlay` recovers. **Prefer resource ids over descriptions**: ids are +stable and not localized. Dump the real tree to find them: + +```bash +adb shell uiautomator dump /sdcard/ui.xml && adb shell cat /sdcard/ui.xml +``` + +### uiflow + +| Function | Use | +|---|---| +| `WaitUI(c, cache, Match, timeout)` | block until a node exists | +| `TapUI(c, cache, Match, timeout)` | wait, then tap its centre | +| `Find(c, cache, Match, timeout)` | get the node for custom handling | +| `WaitPlayback(c, cache, pkg, PlaybackOpts)` | audio session, or a UI fallback signal | +| `Cards(nodes, CardOpts)` | content rows in a dump, top to bottom | +| `WaitCard` / `TapCard` | the nth content row, scrolling if needed | +| `MonkeyLaunch(c, pkg)` | launcher-intent start | +| `CompileRes([]string)` | case-insensitive regexes | + +`Match` combines `TextRE`, `DescRE`, `TextContains`, `DescContains`, +`ResourceContains`, plus `Pick` (`largest`/`smallest`/`first`) and `MinArea`. + +`WaitPlayback` normally confirms via the audio session. Players that start without +one need `SoftFail: true` plus a UI signal (`OrDescContains`, +`OrResourceContains`) so capture proceeds anyway. + +`CardOpts` carries geometry (`MinY`, `MinH`, `MinW`) and denylists (`DescDeny`, +`TextDeny`) so chrome is not mistaken for content. Defaults suit a 1080-wide +phone; put real values in `module.yaml` if your app differs. + +## 4. Optional: catalog + +Implement `app.Catalog` and `drm catalog --app myapp` starts working. + +```go +func (a *App) Resolve(channel string) (session.Stream, error) { + id, err := a.Base.Resolve(channel) + // ... hit the provider API, pick a manifest + license URL ... + return session.Stream{ + Name: Name + "-" + id, App: Name, Channel: id, + MPD: manifestURL, LicenseURL: licenseURL, + HeadersJSON: "{}", Rewriter: "none", ManifestType: "hls", + }, nil +} + +func (a *App) EnrichWithKeys(info *session.Stream, python, wvd string) error { + body, err := brightcove.Get(info.MPD) + // ... extract PSSH, then wvkey.FetchRawAll ... +} +``` + +For a nicer `--list`, also implement `app.CatalogLister` returning +`[]appreg.CatalogRow`. + +Decode remote config generically. Do not model a provider's JSON field names as Go +struct fields — read into `map[string]any` and let `module.yaml` name the fields: + +```yaml +channels: + main: + token_field: liveLoggedOutTokenIOI + stream_id_field: irelandStreamId +``` + +A renamed upstream field then needs an edit to your values file, not a rebuild. + +## 5. Optional: manifest rewriting + +Only needed when the captured manifest expires faster than the stream. + +```go +func init() { + mpd.Register(Name, func() mpd.Rewriter { return NewRewriter() }) +} + +func (r *Rewriter) Name() string { return Name } + +func (r *Rewriter) Rewrite(upstreamXML []byte, kidHint string) ([]byte, error) { + // identify the origin channel from the manifest or the KID, then: + return mpd.BuildSynthetic(r.cfg.Synthetic, channel, originBase, kid) +} + +// Optional: recover the origin when upstream is gone and only the KID remains. +func (r *Rewriter) ResolveKID(kidHex string) (channel, originBase string, ok bool) +func (r *Rewriter) Prime(channel, originBase string) +``` + +Register a **factory**, not an instance — a rewriter may hold per-stream state. + +## 6. Optional: streamd settings + +```go +func (a *App) VideoSelect() string { return "for=best" } +func (a *App) AudioSelect() string { return "lang=en:for=best" } +func (a *App) RewriterName() string { return Name } // or "none" +func (a *App) LiveWaitSeconds() int { return 6 } +func (a *App) TSReadyBytes() int64 { return 2 << 20 } +``` + +Without this, streamd uses neutral defaults. This is what replaced guessing from +app and stream names. + +## 7. Link it in + +```go +// apps/modules/all/all.go +import ( + _ "drmdecryption/modules/myapp" + _ "drmdecryption/modules/rte" + _ "drmdecryption/modules/tg4" +) +``` + +```bash +go -C apps/cli build -o ../../bin/drm . +./bin/drm modules +``` + +## 8. Test it + +Assert the interfaces at compile time and prove no secrets leaked: + +```go +var ( + _ appreg.App = (*App)(nil) + _ appreg.Catalog = (*App)(nil) + _ appreg.StreamDefaults = (*App)(nil) +) + +func TestNoHardcodedCredentials(t *testing.T) { + cfg := Config{}.withDefaults() + if cfg.Package != "" || cfg.APIBase != "" { + t.Fatal("provider values must come from module.yaml") + } +} +``` + +`apps/modules/all/all_test.go` already checks every registered module constructs +with no values file present, reports a valid key mode, and — if it declares a +rewriter name — actually registered one. + +For anything needing the live service or your local values, use a build tag so the +normal run stays hermetic: + +```go +//go:build live +``` + +```bash +go -C apps/modules test -tags live ./myapp/ -v +``` + +## Checklist + +- [ ] No account id, key, token, video id, hostname or KID in any `.go` file +- [ ] `module.yaml` is gitignored (`git check-ignore -v apps/modules/myapp/module.yaml`) +- [ ] `New()` succeeds with no `module.yaml` +- [ ] Launch marker actually matches — watch for a silent full-timeout wait +- [ ] `CaptureHints` asks only for fields the app really produces +- [ ] `KeyMode` is correct (`raw` for binary challenges, `modulardrm` for JSON) +- [ ] Added to `all/all.go` +- [ ] `drm modules` lists it with the right channel count diff --git a/docs/providers/bbc.md b/docs/providers/bbc.md new file mode 100644 index 0000000..c0cc6bc --- /dev/null +++ b/docs/providers/bbc.md @@ -0,0 +1,58 @@ +# BBC module — filling in `module.yaml` + +BBC iPlayer (Android). Streams from the `mobile-phone-main` mediaset are **clear** +(CDN signed URLs); capture only needs the **MPD/HLS master**. Transparent MITM keeps +NordVPN UK on the phone. + +`apps/modules/bbc/module.yaml` is published in the repo (package + channel map only; +no secrets). Skeleton: + +```yaml +package: bbc.iplayer.android +proxy_bin: bin/proxy-android-arm64 +ca_hash: "6c3578b4" + +score: + hosts: + - open.live.bbc.co.uk + - vs-cmaf-push-uk + - vod-dash-uk + - akamaized.net + - bbci.co.uk + - bidi.net.uk + deny: + - telemetry.api.bbci + - bag.api.bbc + - thumbnail + +aliases: + one: bbcone + bbc1: bbcone + +channels: + bbcone: + label: BBC One + vpid: bbc_one_london + bbctwo: + label: BBC Two + vpid: bbc_two_england +``` + +## Capture + +```bash +# Leave NordVPN UK ON. Magisk CA should already be mounted (skip --reinject). +./bin/drm capture --app bbc --channel bbcone --wait 120 +# Play BBC One on the phone when prompted. +``` + +Console prints the MPD in a banner; `outputs/bbc//session.json` holds the +same fields (`mpd` required; no `key` / `pssh` for clear streams). + +Replay: + +```powershell +ffplay -user_agent "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36" -i "" +``` + +PC needs a UK egress IP (VPN). Some CDNs reject HEAD; use GET / ffplay as above. diff --git a/docs/providers/rte.md b/docs/providers/rte.md new file mode 100644 index 0000000..2eec91f --- /dev/null +++ b/docs/providers/rte.md @@ -0,0 +1,224 @@ +# RTE module — filling in `module.yaml` + +DASH + ModularDrm Widevine, captured from the phone. There is no public catalog, so +every channel needs one phone capture to bootstrap. + +Create `apps/modules/rte/module.yaml`. It is gitignored. Nothing below is shipped +in the repo — you obtain it yourself from your own device. + +## Skeleton + +```yaml +package: +license_url: "" +proxy_bin: bin/proxy-android-arm64 +ca_hash: "6c3578b4" + +score: + hosts: + - + - + deny: + - + +aliases: + one: chanone + two: chantwo + +channels: + chanone: + label: "Channel One" + chip_desc: ["^chanone$"] + play_desc: ["play\\s*chanone"] + chantwo: + label: "Channel Two" + chip_desc: ["^chantwo$"] + play_desc: ["play\\s*chantwo"] + +# Origin encoders behind the live channels. +origins: + vc11: "https:///live//vc11/vc11.isml/" + vc12: "https:///live//vc12/vc12.isml/" + +kids: + vc11: "" + vc12: "" +``` + +## Where each value comes from + +### `package` + +Open the app on the phone, then: + +```bash +adb shell dumpsys window | grep mCurrentFocus +``` + +The part before `/` is the package id. + +### `license_url` + +Run a discover session and play a channel ([capture.md](capture.md)): + +```bash +./bin/drm proxy discover --install-ca --reinject +grep -iE 'ModularDrm|widevine|license' outputs/discover/*/appproxy.log +``` + +Take the full URL including its query string — the account reference in it is part +of the endpoint. The module also accepts it per run: + +```bash +./bin/drm capture --app rte --rte.license-url 'https://.../ModularDrm?...' +``` + +### `score.hosts` / `score.deny` + +`hosts` are the hosts that legitimately serve manifests; `deny` are hosts or URL +fragments that must never be mistaken for one. + +```bash +grep -oE '"host":"[^"]+"' outputs/discover/*/appproxy_traffic.jsonl \ + | sort | uniq -c | sort -rn | head -20 +``` + +Put the origin and any ad-stitching host in `hosts`. Put EPG/schedule feeds and +analytics in `deny` — this app's schedule feed otherwise scores as a manifest +because its URL looks plausible. + +Format-level scoring (`.mpd`, `.m3u8`, `/manifest`, `.isml`) is built in; you only +supply the provider-specific parts. + +### `channels.*.chip_desc` and `play_desc` + +These are the regexes that locate the channel chip on the Live tab and the large +hero Play button. They match the node **content-description**, case-insensitively. + +Navigate to the Live tab by hand and dump the tree: + +```bash +adb shell uiautomator dump /sdcard/ui.xml +adb shell cat /sdcard/ui.xml | grep -oE 'content-desc="[^"]*"' | sort -u +``` + +You are looking for two things: + +- a small chip per channel — its description is usually the bare channel name, so + anchor the regex (`^chanone$`) to avoid matching a longer label +- a large "play " button that appears after the chip is tapped + +The module taps the hero Play directly when it is already on screen, otherwise it +taps the chip first and waits for Play to appear. A match is only accepted as the +hero button if its area is at least `hero_min_area` (default 200000), which keeps +small list-item play icons from winning. + +### `origins` and `kids` — the important pair + +These let the manifest rewriter keep working after the ad-stitched manifest session +expires. Both come from a capture. + +**The KID** is in the capture session: + +```bash +./bin/drm capture --app rte --channel chanone --auto-play --wvd data/device.wvd +jq -r '.kid' outputs/rte/latest/session.json +``` + +Convert it to dashed UUID form for the `kids:` map +(`8-4-4-4-12`), e.g. `df1633821dddfdd5bec9822c1ec0f052` becomes +`df163382-1ddd-fdd5-bec9-822c1ec0f052`. Either form is accepted, but UUID form is +easier to read. + +**The origin** is the Smooth Streaming base URL. Find it in the discover traffic: + +```bash +grep -oE 'https?://[^"]+\.isml[^"]*' outputs/discover/*/appproxy_traffic.jsonl \ + | sed -E 's#(\.isml).*#\1/#' | sort -u +``` + +The last path element before `.isml` is the encoder channel id, which becomes the +map key. So `https://host/live/tc-1/vc11/vc11.isml/` is keyed `vc11`. + +**Pair them up** by capturing each channel in turn and noting which KID appeared: + +```bash +for ch in chanone chantwo; do + ./bin/drm capture --app rte --channel $ch --auto-play --wvd data/device.wvd + echo "$ch -> $(jq -r .kid outputs/rte/latest/session.json)" +done +``` + +Then confirm the mapping resolves: + +```bash +go -C apps/modules test -tags live ./rte/ -v +``` + +That test fetches each configured origin's manifest, checks the KID resolves back +to its channel, and verifies the synthetic manifest lands on the live encoder's +segment grid. If a KID does not resolve, `kids:` and `origins:` disagree. + +### `ca_hash` + +The subject hash of the MITM CA, which is what the Magisk mount installs as +`.0`. `6c3578b4` is the bundled CA; change it only if you generated your own. + +```bash +adb shell 'ls /system/etc/security/cacerts/' +``` + +## Optional tuning + +All of these have working defaults — set them only if the app changes. + +```yaml +encoder_channel_re: '^(vc|channel)\d+$' # which origin ids use the encoder timeline +origin_marker: .isml # path element identifying an origin BaseURL +hero_min_area: 200000 # smallest node accepted as the hero Play + +ui: + launch_desc_contains: ["live tab", "header logo"] + live_tab_desc_re: ["Live tab"] + +timeouts: + launch_wait_ui: 25s + live_tab: 30s + chip_wait: 20s + play_wait: 15s + playback: 45s + +synthetic: + video_timescale: 600 + audio_timescale: 48000 + video_name: video + audio_name: audio_128k + video_bandwidth: 6000000 + audio_bandwidth: 128000 + width: 1920 + height: 1080 + live_edge_offset_s: 12 + window_segments: 6 +``` + +The `synthetic` block describes the origin encoder. Its defaults match a common +Smooth-to-DASH setup; if your origin uses different stream names or bitrates, read +them off the origin manifest: + +```bash +curl -s 'https:///live//vc11/vc11.isml/Manifest' | head -40 +``` + +`StreamIndex Name="…"` gives `video_name` / `audio_name`, and `Bitrate="…"` gives +the bandwidths. Segment duration and phase are learned at runtime, not configured. + +## Verify + +```bash +./bin/drm modules # channel count should match your file +./bin/drm capture --app rte --channel chanone --auto-play --wvd data/device.wvd +go -C apps/modules test -tags live ./rte/ -v +``` + +A good capture yields `auth`, `pid`, `pssh` and `mpd`, and a single `KID:KEY` whose +KID matches the `kids:` entry for that channel's encoder. diff --git a/docs/providers/tg4.md b/docs/providers/tg4.md new file mode 100644 index 0000000..5f963eb --- /dev/null +++ b/docs/providers/tg4.md @@ -0,0 +1,288 @@ +# TG4 module — filling in `module.yaml` + +Brightcove Live: HLS manifests and a raw Widevine license. The playback catalog is +public, so channels resolve **without a phone** once you have the account id and +policy key. Phone capture also works and is the only route for channels the catalog +does not list. + +Create `apps/modules/tg4/module.yaml`. It is gitignored. None of these values ship +in the repo. + +## The DRM stack + +| Piece | Detail | +|---|---| +| Manifest | HLS `.m3u8` (prefer the DVR playlist) | +| License | `POST` a raw binary challenge to `.../lic/wv?token=` | +| License auth | JWT in the **query string**, not a header | +| Key mode | `raw` (not the JSON ModularDrm wrapper) | +| Keys per stream | several — a multi-KID SAMPLE-AES/cbcs stream needs all of them | +| PSSH | from the HLS playlist's Widevine `KEYFORMAT` line, not from a manifest | + +Also published on the same host: FairPlay (`/lic/fp`) and PlayReady (`/lic/pr`). + +## Skeleton + +```yaml +package: +proxy_bin: bin/proxy-android-arm64 +ca_hash: "6c3578b4" + +account_id: "" +policy_key: "BCpkAD..." +playback_config_url: "https:////playback_config.json" + +score: + hosts: + - + deny: + - + +aliases: + main: chanone + plus1: plus + +channels: + chanone: + label: "Main" + video_id: "" + stream_id_field: + token_field: + live_card_index: 0 + kids: + label: "Kids" + video_id: "" + token_field: + live_card_index: 1 + sport: + label: "Sport" + phone_only: true # no catalog entry; capture from the phone + live_card_index: 3 + +ui: + launch_resource_contains: + - +``` + +## Where each value comes from + +### `package` + +```bash +adb shell dumpsys window | grep mCurrentFocus +``` + +### `playback_config_url` + +The app fetches a JSON config at startup that carries the live stream ids and +playback tokens. Find it in a discover session: + +```bash +./bin/drm proxy discover --install-ca --reinject +grep -oE 'https?://[^"]+(config|playback)[^"]*\.json' \ + outputs/discover/*/appproxy_traffic.jsonl | sort -u +``` + +Fetch it and look at the keys: + +```bash +curl -s '' | jq 'keys' +``` + +You will see entries along the lines of `StreamId` and +`liveLoggedOutToken`. **Those names go in your channel entries**, not in Go +— the module decodes the config generically, so a renamed upstream field is a +one-line edit here. + +### `account_id` + +The Brightcove account id appears in the catalog request path and in the manifest +URL: + +```bash +grep -oE 'edge\.api\.brightcove\.com/playback/v1/accounts/[0-9]+' \ + outputs/discover/*/appproxy_traffic.jsonl | sort -u +``` + +### `policy_key` + +A `BCpkAD…` string the app sends as `Accept: application/json;pk=` on +every catalog call. It is embedded in the APK and visible on the wire: + +```bash +grep -oE 'pk=BCpkAD[A-Za-z0-9_-]+' outputs/discover/*/appproxy_traffic.jsonl \ + | sort -u +``` + +Treat it as a secret: keep it in this gitignored file, or pass it per run. + +```bash +./bin/drm catalog --app tg4 --channel chanone --tg4.policy-key 'BCpkAD...' +export TG4_POLICY_KEY='BCpkAD...' +``` + +### `channels.*.video_id` vs `stream_id_field` + +Two ways to name the Brightcove video for a channel: + +- `video_id` — pin it directly. Stable, and what you want once you know it. +- `stream_id_field` — read it from `playback_config.json` at run time. Survives the + provider rotating ids. + +Set both and `video_id` wins. Find a channel's id by matching the config field to +the manifest URL you captured — the id appears as the first path segment: + +```text +https:///////playlist-hls-dvr.m3u8 +``` + +### `channels.*.token_field` + +The field in `playback_config.json` holding the long-lived JWT used as +`livePlaybackToken` on the catalog call. Each channel/region has its own. Without +the right one the catalog returns no playable source. + +### `channels.*.live_card_index` + +Only used for phone capture. It is the channel's position in the app's Live page +card list, counting from 0, top to bottom. + +Open the Live page by hand and dump the tree: + +```bash +adb shell uiautomator dump /sdcard/ui.xml +adb shell cat /sdcard/ui.xml +``` + +Content cards are the clickable rows below the title area. The module finds them by +geometry and ignores chrome (Search, Profile, Cast, drawer). Confirm your indices by +running a capture and checking the tap y-coordinate changes between channels: + +```text +[*] tap card #0 @ 540,732 +[*] tap card #1 @ 540,1494 +``` + +If the wanted index is off screen the module scrolls and re-counts, so a long list +still works. + +### `channels.*.phone_only` + +Set it for a channel that has no catalog entry. `drm catalog` then fails with a +message pointing at phone capture instead of a confusing "no video id" error. + +### `ui.launch_resource_contains` + +This app's home screen exposes almost no useful `content-desc` — only chrome like +Search, Profile, Cast and the drawer button — so readiness is keyed off **view +ids**, which are stable and not localized. + +```bash +adb shell uiautomator dump /sdcard/ui.xml +adb shell cat /sdcard/ui.xml | grep -oE 'resource-id="[^"]*"' \ + | sed 's/.*\///' | sort -u +``` + +Pick a container that only exists once the home screen has rendered, e.g. the +content rail's refresh layout or the drawer toolbar. + +This matters: without a marker that actually matches, every capture burns the full +launch timeout before auto-play recovers. It still works, just 30 s slower each time. + +### `score.hosts` / `score.deny` + +`hosts` is the live CDN host that serves manifests. `deny` is for analytics — this +provider's metrics beacons carry `.m3u8` in a query string and otherwise score as +manifests. + +```bash +grep -oE '"host":"[^"]+"' outputs/discover/*/appproxy_traffic.jsonl \ + | sort | uniq -c | sort -rn | head -20 +``` + +## Verify + +Catalog path first — it needs no phone: + +```bash +./bin/drm modules +./bin/drm catalog --app tg4 --list +``` + +```text +CHANNEL CATALOG ID LABEL +chanone 6383454185112 Main +kids 6383455473112 Kids +sport - Sport + +provider flags: cula4Enabled=true liveEnabled=true ... +``` + +A `-` in CATALOG ID means neither `video_id` nor `stream_id_field` resolved. +`provider flags` are the booleans from `playback_config.json`, handy for spotting a +channel the provider has switched off. + +Then resolve one for real: + +```bash +./bin/drm catalog --app tg4 --channel chanone --keys --wvd data/device.wvd +``` + +Expect an HLS DVR manifest, a `/lic/wv?token=` license URL, a PSSH, and several +`KID:KEY` lines. + +Finally, cross-check the phone path against it — independent routes agreeing on the +PSSH and the full key set is strong evidence both are configured correctly: + +```bash +./bin/drm catalog --app tg4 --channel chanone --keys --wvd data/device.wvd --json > a.json +./bin/drm capture --app tg4 --channel chanone --auto-play --wvd data/device.wvd +diff <(jq -r '.keys[]' a.json | sort) <(sort outputs/tg4/latest/key.txt) +``` + +## Optional tuning + +```yaml +stream_name_prefix: tg4 # prefix for generated stream names + +cards: + min_y: 300 # ignore nodes above this (title area / chrome) + min_h: 250 + min_w: 600 + desc_deny: ["Search", "Profile", "Cast", "Open", "Closed"] + text_deny: ["catch-up"] + +ui: + live_label: live # text identifying the Live menu entry and title + live_title_resource: + drawer_desc: ["Open", "Closed"] + drawer_fallback_x: 68 # tapped when the drawer button cannot be found + drawer_fallback_y: 183 + menu_max_x: 700 # bounds for a drawer menu entry + menu_min_y: 250 + menu_max_y: 700 + playback_desc_contains: ["video player"] + playback_resource_contains: [] + +timeouts: + launch_wait_ui: 30s + live_nav: 25s + card: 25s + playback: 60s +``` + +The player often starts without a dumpsys audio session, so playback is confirmed +by `playback_resource_contains` / `playback_desc_contains` and the wait is +soft-failing: capture continues even if neither signal appears. + +## Troubleshooting + +| Symptom | Cause | +|---|---| +| `INVALID_POLICY_KEY` (HTTP 401) | wrong or rotated `policy_key` | +| `no playback token for channel "x" (field Y)` | `token_field` is not a key in `playback_config.json` | +| `no video id for channel "x"` | set `video_id`, or a `stream_id_field` that exists | +| `no HLS source in playback response` | the channel is off air, or the token is for a different region | +| `launch: ui node not found` | `ui.launch_resource_contains` does not match — re-dump the tree | +| `tap card #N` picks the wrong channel | card order changed; re-check `live_card_index` | +| Fewer keys than expected | a multi-KID stream needs every key; check `key.txt`, not just `key` | diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..17e4538 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,252 @@ +# Quickstart + +From an empty directory to a captured Widevine key. + +## 1. Prerequisites + +| Need | Version | Needed for | +|---|---|---| +| Go | 1.25+ | everything | +| Python | 3.10+ | the CDM helper (`apps/wvkey/wvkey.py`) | +| adb | any recent platform-tools, on `PATH` | phone capture only | +| `.wvd` device file | — | fetching content keys | +| Rooted Android phone (Magisk) | — | phone capture only | + +Check Go: + +```bash +go version +``` + +If Go is installed but not on `PATH`, set `GOROOT` and prepend its `bin`: + +```bash +export GOROOT=/path/to/go +export PATH="$GOROOT/bin:$PATH" +``` + +## 2. Clone and build + +```bash +git clone https://git.nulldrm.com/nulldrm/Null-DRM-Official.git +cd Null-DRM-Official + +go -C apps/cli build -o ../../bin/drm . # Windows: -o ../../bin/drm.exe +``` + +That single command builds the whole toolchain: capture, catalog, agent, the +streamd control plane, the MITM host CLI, and every app module. + +Verify: + +```bash +./bin/drm help +./bin/drm modules +``` + +Expected on a fresh clone — modules exist, values do not: + +```text +APP PACKAGE KEY MODE REWRITER CHANNELS VALUES +rte (not configured) modulardrm rte 0 missing: .../apps/modules/rte/module.yaml +tg4 (not configured) raw none 0 missing: .../apps/modules/tg4/module.yaml +``` + +## 3. The Python CDM helper + +Only `wvkey.py` is Python; it turns a PSSH + license URL into `KID:KEY` lines. + +```bash +python -m venv .venv +.venv/bin/pip install -r apps/wvkey/requirements.txt +``` + +On Windows: + +```powershell +python -m venv .venv +.venv\Scripts\pip install -r apps\wvkey\requirements.txt +``` + +The binary finds `.venv` automatically; override with `--python /path/to/python`. + +Place your Widevine device file under `data/` (gitignored): + +```text +data/device.wvd +``` + +## 4. Add values for a module + +Create the module's values file. It is gitignored, so it never leaves your machine. + +```bash +$EDITOR apps/modules/tg4/module.yaml +``` + +Minimum for TG4 — see [providers/tg4.md](providers/tg4.md) for where each comes from: + +```yaml +package: +account_id: "" +policy_key: "BCpkAD..." +playback_config_url: "https://.../playback_config.json" + +channels: + ioi: + label: "Main channel" + video_id: "" + token_field: + live_card_index: 0 +``` + +Confirm it loaded: + +```bash +./bin/drm modules +./bin/drm catalog --app tg4 --list +``` + +Nothing stored? Pass values per run instead: + +```bash +./bin/drm catalog --app tg4 --channel ioi \ + --tg4.account-id 1234567890 --tg4.policy-key 'BCpkAD...' +``` + +Or through the environment (`_`, upper snake case): + +```bash +export TG4_POLICY_KEY='BCpkAD...' +``` + +Precedence is **flag → environment → module.yaml → built-in default**. Only +mechanical defaults (timeouts, DASH timescales, card geometry) have built-ins. + +## 5. Get keys without a phone + +If the app has a public playback catalog: + +```bash +./bin/drm catalog --app tg4 --channel ioi --keys --wvd data/device.wvd +``` + +```text +manifest: https://.../playlist-hls-dvr.m3u8 +license_url: https://.../lic/wv?token=... +pssh: AAAAYHBzc2gAAAAA7e+LqXnWSs6j... +key: 27c38b955b983ac2827204f8d260d628:f31efbc58fc89526cce5e5dee50a7ebd +key[0]: 5b9d58edf81f3b5b9c3cc61f3e81845e:688236976dcf8cdde5cdfac4a698f256 +... +``` + +Add `--json` for a machine-readable object, `--out` to write a session directory +without fetching keys. + +## 6. Get keys from the phone + +One-time device setup: + +```bash +adb devices # your phone must be listed and authorised +./bin/drm proxy build # cross-compiles the on-device MITM (linux/arm64) +./bin/drm proxy install-ca --reinject +``` + +`install-ca` pushes the MITM CA and Magisk-mounts it into the system trust store. +Without it apps fail TLS with `unknown certificate`. See [capture.md](capture.md) +if the reinject does not report success. + +Then capture: + +```bash +./bin/drm capture --app rte --channel rteone --auto-play --wvd data/device.wvd +``` + +What it does: start the on-device MITM → point the phone's HTTP proxy at it → +reinject the CA → launch the app → drive its UI to the channel → wait for the +license/manifest traffic → run the CDM → write a session → close the app. + +```text +[*] Launching rte (...) +[*] Auto-play RTE One… +[*] tap_ui "Live"/"Live tab, 2 out of 5" @ 324,2253 +[*] tap Play @ 540,720 +[+] ... is PLAYING +[+] Capture: + pid: 8T8Ohq7XT5KN + mpd: https://dai.google.com/linear/dash/pa/event/... +[+] key: df1633821dddfdd5bec9822c1ec0f052:b6324f76d2bb4dfe63e4272f803274f4 +[+] session: outputs/rte/20261005-215448/session.json +``` + +Keep the screen on for the whole run: + +```bash +adb shell svc power stayon true +``` + +Drop `--auto-play` to launch the app and navigate by hand, or drop `--app` +entirely for passive mode — the MITM records whatever you play. + +## 7. Control plane + +```bash +./bin/drm serve --bind 127.0.0.1:8083 --data .cache/streamd --token SECRET +``` + +Open . The app and channel pickers are populated from +`GET /api/apps`, i.e. from the modules compiled into the binary. + +Create a stream from a capture: + +```bash +./bin/drm catalog --app tg4 --channel ioi --keys --wvd data/device.wvd --json \ + > /tmp/s.json + +curl -X POST http://127.0.0.1:8083/api/streams \ + -H 'Authorization: Bearer SECRET' -H 'Content-Type: application/json' \ + --data @/tmp/s.json +``` + +## 8. Agent + +The agent polls streamd, queues any enabled stream that is down or missing +credentials, claims a free phone, re-captures it and POSTs the result back. + +```bash +export STREAMD_TOKEN=SECRET +./bin/drm agent run --config apps/agent/agent.yaml + +./bin/drm agent devices # which phones it can use +./bin/drm agent status # recent jobs +./bin/drm agent enqueue --stream tg4-ioi --reason manual +``` + +Details in [streamd.md](streamd.md). + +## 9. Tests + +```bash +go -C apps/pkg test ./... +go -C apps/modules test ./... +go -C apps/streamd test ./... +``` + +Live-origin tests are tag-gated and need your local values: + +```bash +go -C apps/modules test -tags live ./rte/ -v +``` + +## Troubleshooting + +| Symptom | Cause | +|---|---| +| `unknown app "x"` | not compiled in — check `apps/modules/all/all.go`, then rebuild | +| `channels 0` in `drm modules` | no `module.yaml`, or `channels:` is empty | +| `unknown channel "y"` | id not in `channels:` and not in `aliases:` | +| `tls: unknown certificate` on the phone | CA not mounted — `drm proxy install-ca --reinject` | +| `wvkey requires --wvd` | pass `--wvd path/to/device.wvd` | +| `timed out waiting for capture` | playback never started, or the app used a path the MITM missed — see [capture.md](capture.md) | +| `INVALID_POLICY_KEY` | wrong or stale `policy_key` | diff --git a/docs/streamd.md b/docs/streamd.md new file mode 100644 index 0000000..faef882 --- /dev/null +++ b/docs/streamd.md @@ -0,0 +1,183 @@ +# Control plane and agent + +Two long-running pieces: `drm serve` holds the state and the dashboard, `drm agent +run` keeps credentials fresh using real phones. + +## drm serve + +```bash +./bin/drm serve --bind 127.0.0.1:8083 --data .cache/streamd --token SECRET +``` + +| Flag | Default | Purpose | +|---|---|---| +| `--bind` | `0.0.0.0:8083` | listen address | +| `--data` | `.cache/streamd` | SQLite, work dirs, HLS output, logs | +| `--token` | `$STREAMD_TOKEN` | bearer token for mutating routes | +| `--nre` | `$NRE_PATH`, `bin/`, `PATH` | downloader binary | +| `--ffmpeg` | `$FFMPEG_PATH`, `bin/`, `PATH` | ffmpeg | +| `--mp4decrypt` | `$MP4DECRYPT_PATH`, `bin/`, `PATH` | decrypter | + +Open . GET routes are unauthenticated; anything mutating +needs `Authorization: Bearer `. + +### API + +| Route | Purpose | +|---|---| +| `GET /api/health` | liveness + uptime | +| `GET /api/apps` | compiled-in modules, their channels, registered rewriters | +| `GET /api/streams` | all streams with runtime state | +| `POST /api/streams` | create | +| `PATCH /api/streams/:id` | update | +| `DELETE /api/streams/:id` | remove | +| `POST /api/streams/:id/start\|stop\|restart` | desired state | +| `POST /api/streams/:id/credentials` | push a fresh manifest + keys | +| `POST /api/streams/:id/claim` | agent claims the stream | + +The dashboard's app and channel pickers are filled from `/api/apps`, so they reflect +whatever modules the binary was built with. Nothing about a provider is hardcoded in +the UI or the schema. + +### Creating a stream + +```bash +./bin/drm catalog --app tg4 --channel ioi --keys --wvd data/device.wvd --json > s.json + +curl -X POST http://127.0.0.1:8083/api/streams \ + -H 'Authorization: Bearer SECRET' -H 'Content-Type: application/json' \ + --data @s.json +``` + +A stream created without `app`, `video_select`, `audio_select` or `rewriter` keeps +those empty on purpose. They are resolved when the stream starts: + +```text +stored row -> the stream's app module (app.StreamDefaults) -> neutral defaults +``` + +So a module decides its own downloader settings, an operator override in the row +still wins, and an unknown app gets something harmless rather than another +provider's preferences. + +### Refreshing credentials + +```bash +curl -X POST http://127.0.0.1:8083/api/streams/1/credentials \ + -H 'Authorization: Bearer SECRET' -H 'Content-Type: application/json' \ + -d '{"mpd":"https://...","key":"KID:KEY\nKID:KEY","headers_json":"{}"}' +``` + +This is exactly what the agent does after a successful capture. + +### Manifest rewriting + +If a stream's `rewriter` names a registered rewriter and the manifest is not HLS, +the supervisor starts a local rewrite server and points the downloader at it: + +```text +stream rewriter="rte" -> mpd.Lookup("rte") -> http://127.0.0.1:PORT/manifest.mpd +``` + +The log line is `stream X: rewritten MPD http://127.0.0.1:…`. Streamd never imports +a provider package to do this — the module registered the rewriter under that name +at init time. + +## drm agent run + +```bash +export STREAMD_TOKEN=SECRET +./bin/drm agent run --config apps/agent/agent.yaml +``` + +What it does, in a loop: + +1. poll streamd for enabled streams that are down or missing credentials +2. enqueue a job in a durable SQLite queue +3. claim a free ADB device that already has the app installed +4. run a phone capture through the stream's app module +5. `POST /api/streams/:id/credentials` +6. force-stop the app and release the device + +It never installs apps and never runs a downloader. A websocket heartbeat keeps its +claims alive, so claims expire automatically if the agent dies. + +### Subcommands + +```bash +./bin/drm agent devices # phones it can use +./bin/drm agent status # recent jobs +./bin/drm agent enqueue --stream tg4-ioi --reason manual +./bin/drm agent cancel --job 3 +``` + +### Config + +`apps/agent/agent.yaml`: + +```yaml +streamd_url: http://127.0.0.1:8083 +# token: taken from $STREAMD_TOKEN when unset +poll_interval_sec: 30 +data_dir: .cache/agent +workers: 1 +max_attempts: 5 +capture_wait_sec: 180 +# devices: [SERIAL1, SERIAL2] # empty = any authorised device +# python: path to python for wvkey.py +# wvd: data/device.wvd +# user_agent: "" +# agent_id: defaults to hostname +``` + +There is no modules directory to configure — app modules are compiled into the +binary. + +`wvd` must be set (here or on the phone capture) or key fetches fail. + +## Running both locally + +```bash +# terminal 1 +./bin/drm serve --bind 127.0.0.1:8083 --data .cache/streamd --token SECRET + +# terminal 2 +export STREAMD_TOKEN=SECRET +./bin/drm agent run --config apps/agent/agent.yaml +``` + +Then create a stream, enable it, and watch the agent pick it up: + +```bash +./bin/drm agent status +``` + +## External tools + +The media supervisor shells out to three binaries. Put them on `PATH`, in `bin/`, +or name them with flags/env: + +| Tool | Flag | Env | +|---|---|---| +| N_m3u8DL-RE | `--nre` | `NRE_PATH`, `N_M3U8DL_RE` | +| ffmpeg | `--ffmpeg` | `FFMPEG_PATH`, `FFMPEG` | +| mp4decrypt | `--mp4decrypt` | `MP4DECRYPT_PATH`, `MP4DECRYPT` | + +Streams can be created, claimed and credentialed without any of them; only actually +pulling media needs them. The supervisor is the least exercised part of the system — +treat its health reporting as control-plane bookkeeping. + +## Data + +```text +.cache/streamd/ + streamd.db SQLite: streams, runtime, claims + work// downloader scratch + keys.txt + www// HLS output + logs/ per-stream downloader + ffmpeg logs + +.cache/agent/ + queue.db durable job queue +``` + +Back up `streamd.db` before upgrading — schema defaults change between versions. diff --git a/www/docs.html b/www/docs.html new file mode 100644 index 0000000..b96b400 --- /dev/null +++ b/www/docs.html @@ -0,0 +1,82 @@ + + + + + + Docs — NULLDRM + + + + + + + + +
+
+ + + NULLDRM + + +
+
+ +
+
+
+
+ +
+
+ README.md + View on Forgejo +
+
Loading document…
+ +
+
+
+
+
+ + + + + + + + + + diff --git a/www/docs.js b/www/docs.js new file mode 100644 index 0000000..dd9d727 --- /dev/null +++ b/www/docs.js @@ -0,0 +1,163 @@ +(() => { + const OWNER = "nulldrm"; + const REPO = "Null-DRM-Official"; + const BRANCH = "main"; + const API = "/git-api/api/v1/repos/" + OWNER + "/" + REPO; + const FORGEJO = "https://git.nulldrm.com/" + OWNER + "/" + REPO; + const DEFAULT_DOC = "README.md"; + const FALLBACK = [ + "README.md", + "quickstart.md", + "architecture.md", + "modules.md", + "capture.md", + "streamd.md", + "providers/rte.md", + "providers/tg4.md", + "providers/bbc.md", + ]; + // Order the nav the way the docs are meant to be read. + const ORDER = FALLBACK; + + const listEl = document.getElementById("docs-list"); + const contentEl = document.getElementById("docs-content"); + const statusEl = document.getElementById("docs-status"); + const titleEl = document.getElementById("docs-title"); + const sourceEl = document.getElementById("docs-source"); + function currentFile() { + const params = new URLSearchParams(window.location.search); + const f = (params.get("f") || DEFAULT_DOC).trim(); + // Allow one level of subdirectory (providers/tg4.md) but nothing escaping docs/. + if (!f || f.includes("..") || f.includes("\\")) return DEFAULT_DOC; + if (!/^[\w.-]+(?:\/[\w.-]+)?\.md$/i.test(f)) return DEFAULT_DOC; + return f; + } + + // Keep "/" readable in the path; only the segments need escaping. + function encodePath(name) { + return name.split("/").map(encodeURIComponent).join("/"); + } + + function forgejoSrc(name) { + return FORGEJO + "/src/branch/" + BRANCH + "/docs/" + encodePath(name); + } + + function forgejoRaw(name) { + return "/git-api/" + OWNER + "/" + REPO + "/raw/branch/" + BRANCH + "/docs/" + encodePath(name); + } + + function setActiveNav(name) { + if (!listEl) return; + listEl.querySelectorAll("a").forEach((a) => { + a.classList.toggle("active", a.getAttribute("data-file") === name); + }); + } + + function renderNav(files) { + if (!listEl) return; + listEl.innerHTML = files.map((name) => { + const href = "docs.html?f=" + encodeURIComponent(name); + return "
  • " + name + "
    • "; + }).join(""); + } + + function rewriteDocLinks(root) { + root.querySelectorAll("a[href]").forEach((a) => { + const href = a.getAttribute("href") || ""; + if (/^https?:\/\//i.test(href) || href.startsWith("#") || href.startsWith("mailto:")) return; + const clean = href.replace(/^\.\//, ""); + if (/^[\w.-]+(?:\/[\w.-]+)?\.md(?:#.*)?$/i.test(clean)) { + const [file, hash] = clean.split("#"); + a.setAttribute("href", "docs.html?f=" + encodeURIComponent(file) + (hash ? "#" + hash : "")); + } + }); + } + + async function dirDocs(path) { + const res = await fetch(API + "/contents/" + path + "?ref=" + encodeURIComponent(BRANCH)); + if (!res.ok) throw new Error("HTTP " + res.status); + const entries = await res.json(); + return Array.isArray(entries) ? entries : []; + } + + async function listDocs() { + try { + const entries = await dirDocs("docs"); + const files = entries + .filter((e) => e && e.type === "file" && /\.md$/i.test(e.name || "")) + .map((e) => e.name); + + // Pull in one level of subdirectories, e.g. docs/providers/. + const dirs = entries.filter((e) => e && e.type === "dir" && e.name); + for (const d of dirs) { + try { + const sub = await dirDocs("docs/" + d.name); + sub + .filter((e) => e && e.type === "file" && /\.md$/i.test(e.name || "")) + .forEach((e) => files.push(d.name + "/" + e.name)); + } catch { + /* a subdirectory we cannot list is not fatal */ + } + } + + files.sort((a, b) => { + const ia = ORDER.indexOf(a); + const ib = ORDER.indexOf(b); + if (ia !== -1 || ib !== -1) { + return (ia === -1 ? ORDER.length : ia) - (ib === -1 ? ORDER.length : ib); + } + return a.localeCompare(b); + }); + return files.length ? files : FALLBACK; + } catch { + return FALLBACK; + } + } + + async function loadDoc(name) { + if (titleEl) titleEl.textContent = name; + if (sourceEl) { + sourceEl.href = forgejoSrc(name); + sourceEl.textContent = "View on Forgejo"; + } + setActiveNav(name); + if (statusEl) { + statusEl.hidden = false; + statusEl.textContent = "Loading " + name + "…"; + } + if (contentEl) contentEl.hidden = true; + + try { + const res = await fetch(forgejoRaw(name)); + if (!res.ok) throw new Error("HTTP " + res.status); + const md = await res.text(); + if (typeof marked === "undefined" || !marked.parse) { + throw new Error("marked missing"); + } + marked.setOptions({ gfm: true, breaks: false }); + contentEl.innerHTML = marked.parse(md); + rewriteDocLinks(contentEl); + statusEl.hidden = true; + contentEl.hidden = false; + document.title = name + " — NULLDRM Docs"; + } catch { + if (statusEl) { + statusEl.innerHTML = + "Could not load " + name + ". " + + "Open it on Forgejo."; + statusEl.hidden = false; + } + if (contentEl) contentEl.hidden = true; + } + } + + async function boot() { + const files = await listDocs(); + renderNav(files); + let file = currentFile(); + if (files.indexOf(file) === -1) file = files[0] || DEFAULT_DOC; + await loadDoc(file); + } + + boot(); +})(); diff --git a/www/index.html b/www/index.html new file mode 100644 index 0000000..8e26ba3 --- /dev/null +++ b/www/index.html @@ -0,0 +1,326 @@ + + + + + + NULLDRM — capture · decrypt · restream, self-hosted + + + + + + + + +
    +
    + + + NULLDRM + + +
    +
    + +
    + +
    +
    + Free forever · open source +

    Capture, decrypt and restream your own subscriptions.

    +

    + NULLDRM logs the Widevine handshake from an Android app, recovers the content keys, + and keeps a live restream healthy. One binary, self-hosted, no license keys. +

    +
    + Quick start + Read the docs + Source +
    +
    +
    + +
    +
    +
    + How it works +

    A capture, start to finish

    +

    + An HTTPS proxy runs on the phone. NULLDRM drives the real app to a channel, + reads the licence exchange as it happens, and resolves the keys with your own CDM. +

    +
    + +
    +
    + + + + drm capture — rteone +
    +
    PS C:\DRM-Decryption> ./bin/drm capture --app rte --channel rteone --auto-play
+[*] Stopping any old appproxy...
+[*] Pushing appproxy...
+20:30:45 CA cert: /data/local/tmp/appproxy-ca.crt
+20:30:45 capture → /data/local/tmp/appproxy_cap.json
+20:30:45 DNS → [1.1.1.1:53 1.0.0.1:53 8.8.8.8:53]
+20:30:45 listening on [::]:8080
+[+] appproxy pid=12004; capture -> /data/local/tmp/appproxy_cap.json
+[*] Setting HTTP proxy -> 127.0.0.1:8080 (was "null")
+[*] Re-injecting system CA (Magisk)...
+[+] System CA present in conscrypt
+[*] Launching rte (package from module.yaml)…
+[*] Auto-play RTE One…
+[*] tap_ui "Live"/"Live tab, 2 out of 5" @ 324,2253
+[*] tap Play @ 540,720
+[+] player is PLAYING
+[*] Waiting for license + PSSH + manifest…
+    watch: .cache\appproxy-default.log
+[+] Capture:
+    pid: ••••••••••••
+    mpd: https://dai.google.com/linear/dash/pa/event/••••/stream/••••
+    license: https://widevine.entitlement.••••/wv/web/ModularDrm?schema=1.0&form=json
+    pssh: AAAASnBzc2gAAAAA7e+LqXnWSs6jyCfc1R0h7QAAACoiIG••••…
+[+] key: ••••••••••••••••••••••••••••••••:••••••••••••••••••••••••••••••••
+[+] session: outputs\rte\20261005-203112\session.json
+[*] Closing player…
+[*] HTTP proxy cleared
    +
    +

    + Redacted above: the package id, licence account, programme id, PSSH and keys. + Those belong to your device and your subscription — NULLDRM keeps them in a local + file that is never committed. +

    + +
    +
    +

    Nothing is cracked

    +

    + Keys come from a Widevine device file you supply. NULLDRM automates the request + flow a normal client already performs — it does not break the DRM. +

    +
    +
    +

    Catalogue or phone

    +

    + Some apps publish a playback catalogue, so a channel resolves with no phone at all. + Everything else goes through the on-device proxy. +

    +
    +
    +

    Stale keys heal

    +

    + Live keys rotate. The agent notices a dead channel, re-runs the capture on a + phone and posts fresh credentials back to the restreamer. +

    +
    +
    +
    +
    + +
    +
    +
    + Ways to run it +

    Start small, grow if you want to

    +

    Same binary in all three. You decide how much automation and how much cooperation.

    +
    + +
    +
    + +

    On your desk

    +

    + Run a capture when you need fresh material, decrypt locally, play it back. + No server, nothing always-on. +

    +
      +
    • One-shot captures
      • +
    • No VPS required
      • +
    +
    + +
    + +

    Server plus a phone

    +

    + Host the restreamer so channels stay up. Leave a phone plugged into the agent + and it refreshes keys on its own when something dies. +

    +
      +
    • Dashboard and API on :8083
      • +
    • Agent watches health on a loop
      • +
    +
    + +
    + +

    Join a swarm

    +

    + A host publishes the channel list. Contributors point agents at it and cover + only the channels they actually subscribe to. +

    +
      +
    • Nobody needs every platform
      • +
    • Any matching agent can heal a feed
      • +
    +
    +
    + +

    + That is the point of a swarm: distributed ownership. One person + should not need every subscription. Many agents, many logins, one shared restream. +

    +
    +
    + +
    +
    +
    + Quick start +

    Four commands

    +

    Go 1.25+ and Python 3.10+. A phone is only needed for the capture step.

    +
    + +
    # 1. build the one binary
+git clone https://git.nulldrm.com/nulldrm/Null-DRM-Official
+go -C apps/cli build -o ../../bin/drm .
+
+# 2. the CDM helper
+python -m venv .venv && .venv/bin/pip install -r apps/wvkey/requirements.txt
+
+# 3. see which app modules are compiled in
+./bin/drm modules
+
+# 4. capture a channel
+./bin/drm capture --app rte --channel rteone --auto-play --wvd data/device.wvd
    + +

    + A fresh clone ships no account ids, keys or hostnames — module code is public, + its values are not. The quickstart walks + through filling in your own, and + the capture guide shows how to discover them + for an app nobody has written a module for yet. +

    +
    +
    + +
    +
    +
    + What is inside +

    One binary, a handful of parts

    +

    Everything ships as drm. Each subcommand is one of these.

    +
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    PartWhat it does
    drm captureDrives the phone through the on-device proxy and writes a session: manifest, PSSH, keys.
    drm catalogResolves a channel from a public playback catalogue, no phone involved.
    drm serveControl plane: REST API, SQLite, dashboard, and the restream supervisor.
    drm agentPolls the control plane, claims a free phone and refreshes dead channels.
    drm proxyBuilds and installs the on-device HTTPS proxy, and records unknown apps.
    drm modulesLists the app modules this build contains and where each one reads its values.
    +
    + +

    + An app module is a small Go package plus a local values file. Adding one means a new + package and a single import line — no fork, no plugin runtime. + See writing a module. +

    +
    +
    + +
    +
    +
    + Source +

    git.nulldrm.com

    +

    + Open source, free forever: no paid tiers, no licence keys, no feature gates. + The official source is Forgejo at nulldrm/Null-DRM-Official. +

    +
    + +
    +
    —Stars
    +
    —Forks
    +
    —Issues
    +
    —Updated
    +
    + +
    + Open Forgejo + Releases + Telegram +
    + +

    Latest releases

    +
    Loading releases…
    + +
    +
    + +
    + +
    +
    + nulldrm.com — self-hosted, free forever. + +
    +
    +

    Use only on services you are authorised to access, with content you have the right to decrypt.

    +
    +
    + + + + + + diff --git a/www/script.js b/www/script.js new file mode 100644 index 0000000..3960a2a --- /dev/null +++ b/www/script.js @@ -0,0 +1,192 @@ +(() => { + const OWNER = "nulldrm"; + const REPO = "Null-DRM-Official"; + const API = "/git-api/api/v1/repos/" + OWNER + "/" + REPO; + const FORGEJO = "https://git.nulldrm.com/" + OWNER + "/" + REPO; + const TG_URL = "https://t.me/+GZwQ7d_HD2w2ZTM8"; + + function setText(id, value) { + const el = document.getElementById(id); + if (el) el.textContent = value; + } + + function shortDate(iso) { + if (!iso) return "—"; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return "—"; + return d.toLocaleDateString(undefined, { year: "numeric", month: "short", day: "numeric" }); + } + + function escapeHtml(s) { + return String(s) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); + } + + async function loadRepo() { + if (!document.getElementById("git-stats")) return; + try { + const res = await fetch(API); + if (!res.ok) throw new Error("HTTP " + res.status); + const data = await res.json(); + setText("stat-stars", String(data.stars_count ?? data.stars ?? 0)); + setText("stat-forks", String(data.forks_count ?? data.forks ?? 0)); + setText("stat-issues", String(data.open_issues_count ?? data.open_issues ?? 0)); + setText("stat-updated", shortDate(data.updated_at || data.pushed_at)); + } catch { + ["stat-stars", "stat-forks", "stat-issues", "stat-updated"].forEach((id) => setText(id, "—")); + } + } + + async function loadReleases() { + const status = document.getElementById("releases-status"); + const list = document.getElementById("releases-list"); + if (!status || !list) return; + + try { + const res = await fetch(API + "/releases?limit=5"); + if (!res.ok) throw new Error("HTTP " + res.status); + const releases = await res.json(); + if (!Array.isArray(releases) || releases.length === 0) { + status.textContent = "No releases published yet — clone the repo or watch it on Forgejo."; + status.hidden = false; + list.hidden = true; + return; + } + + list.innerHTML = releases.map((r) => { + const title = escapeHtml(r.name || r.tag_name || "Release"); + const tag = escapeHtml(r.tag_name || ""); + const when = shortDate(r.published_at || r.created_at); + const url = escapeHtml(r.html_url || (FORGEJO + "/releases/tag/" + encodeURIComponent(r.tag_name || ""))); + const body = (r.body || "").trim(); + const preview = body + ? '

    ' + escapeHtml(body.slice(0, 180)) + (body.length > 180 ? "…" : "") + "

    " + : ""; + const assets = Array.isArray(r.assets) ? r.assets : []; + const assetLinks = assets.slice(0, 4).map((a) => { + const href = escapeHtml(a.browser_download_url || a.html_url || "#"); + const name = escapeHtml(a.name || "asset"); + return '' + name + ""; + }).join(" · "); + return ( + "
  • " + + '

    ' + title + "

    " + + '

    ' + tag + (tag ? " · " : "") + when + + (assetLinks ? " · " + assetLinks : "") + "

    " + + preview + + "
    • " + ); + }).join(""); + + status.hidden = true; + list.hidden = false; + } catch { + status.innerHTML = + 'Could not load releases. Open Forgejo releases directly.'; + status.hidden = false; + list.hidden = true; + } + } + + // Telegram invite: shown once per browser. + function initTelegramInvite() { + const KEY = "nulldrm_tg_seen"; + const modal = document.getElementById("tg-modal"); + if (!modal) return; + + let seen = false; + try { + seen = localStorage.getItem(KEY) === "1"; + } catch { + seen = true; // private mode: do not nag + } + + const markSeen = () => { + try { + localStorage.setItem(KEY, "1"); + } catch { + /* private mode */ + } + }; + + // Belt and braces: the hidden attribute plus an inline display, so the + // overlay cannot survive a stylesheet that sets display on .tg-modal. + const close = () => { + markSeen(); + modal.hidden = true; + modal.style.display = "none"; + document.body.style.overflow = ""; + }; + + const open = () => { + modal.hidden = false; + modal.style.display = "grid"; + document.body.style.overflow = "hidden"; + const join = document.getElementById("tg-join"); + if (join) join.focus(); + }; + + // Wire every dismiss control directly rather than relying on delegation. + modal.querySelectorAll("[data-tg-dismiss]").forEach((el) => { + el.addEventListener("click", (e) => { + e.preventDefault(); + close(); + }); + }); + + // Clicking anywhere outside the dialog also closes it. + modal.addEventListener("click", (e) => { + const t = e.target; + if (t instanceof Element && !t.closest(".tg-dialog")) close(); + }); + + const join = document.getElementById("tg-join"); + if (join) join.addEventListener("click", close); + + document.addEventListener("keydown", (e) => { + if (e.key === "Escape" && !modal.hidden) close(); + }); + + if (seen) { + modal.hidden = true; + modal.style.display = "none"; + } else { + open(); + } + } + + // Highlight the section currently in view. + function initNavHighlight() { + const links = Array.from(document.querySelectorAll('.site-nav a[href^="#"]')); + if (!links.length || !("IntersectionObserver" in window)) return; + const byId = new Map(links.map((a) => [a.getAttribute("href").slice(1), a])); + const sections = Array.from(byId.keys()) + .map((id) => document.getElementById(id)) + .filter(Boolean); + if (!sections.length) return; + + const obs = new IntersectionObserver( + (entries) => { + entries.forEach((entry) => { + const link = byId.get(entry.target.id); + if (link) link.classList.toggle("active", entry.isIntersecting); + }); + }, + { rootMargin: "-45% 0px -50% 0px" } + ); + sections.forEach((s) => obs.observe(s)); + } + + loadRepo(); + loadReleases(); + initTelegramInvite(); + initNavHighlight(); + + // Any Telegram link left without an href falls back to the canonical invite. + document.querySelectorAll('a[href^="https://t.me/"]').forEach((a) => { + if (!a.getAttribute("href")) a.setAttribute("href", TG_URL); + }); +})(); diff --git a/www/styles.css b/www/styles.css new file mode 100644 index 0000000..04644d8 --- /dev/null +++ b/www/styles.css @@ -0,0 +1,381 @@ +/* NULLDRM - flat surfaces, soft depth, one accent. Light only. */ + +:root { + color-scheme: light; + + --accent: #4c6ef5; + --accent-ink: #ffffff; + --accent-soft: #eef1fe; + --accent-line: #c9d4fb; + + /* white with a slight grey tint */ + --bg: #f6f7f9; + --surface: #ffffff; + --surface-2: #f0f2f5; + --ink: #15181d; + --ink-2: #434a53; + --muted: #6b7280; + --line: #e4e7ec; + --line-strong: #d4d9e0; + + --ok: #1a7f52; + --warn: #9a6a00; + + /* the terminal card stays dark on purpose - it is a terminal */ + --term-bg: #15181f; + --term-ink: #d9dee8; + --term-dim: #7b8595; + --term-ok: #6fd79b; + --term-url: #8fb6ff; + --term-redact: #5c6678; + + --radius: 12px; + --radius-sm: 8px; + --shadow: 0 1px 2px rgba(16, 20, 30, 0.05), 0 6px 20px -8px rgba(16, 20, 30, 0.12); + --shadow-sm: 0 1px 2px rgba(16, 20, 30, 0.06); + + --maxw: 1080px; + --sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Inter, Roboto, Helvetica, Arial, sans-serif; + --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace; +} + +/* The hidden attribute must win over any display rule below. */ +[hidden] { display: none !important; } + +* { box-sizing: border-box; } + +html { scroll-behavior: smooth; } +@media (prefers-reduced-motion: reduce) { html { scroll-behavior: auto; } } + +body { + margin: 0; + background: var(--bg); + color: var(--ink); + font: 16px/1.65 var(--sans); + -webkit-font-smoothing: antialiased; +} + +h1, h2, h3 { line-height: 1.2; letter-spacing: -0.015em; margin: 0; } +h1 { font-size: clamp(2rem, 5vw, 3rem); font-weight: 680; } +h2 { font-size: clamp(1.3rem, 3vw, 1.6rem); font-weight: 650; } +h3 { font-size: 1.02rem; font-weight: 640; } +p { margin: 0; } + +a { color: var(--accent); text-decoration: none; } +a:hover { text-decoration: underline; } + +code, pre, kbd { font-family: var(--mono); } +code { + font-size: 0.88em; + background: var(--surface-2); + border: 1px solid var(--line); + border-radius: 5px; + padding: 0.1em 0.38em; +} + +.skip { + position: absolute; + left: -9999px; +} +.skip:focus { + left: 12px; top: 12px; z-index: 100; + background: var(--surface); color: var(--ink); + padding: 10px 14px; border: 1px solid var(--line-strong); border-radius: var(--radius-sm); +} + +:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } + +.shell { max-width: var(--maxw); margin: 0 auto; padding: 0 20px; } + +/* ---------- header ---------- */ + +.site-header { + position: sticky; top: 0; z-index: 20; + background: rgba(246, 247, 249, 0.85); + backdrop-filter: saturate(140%) blur(10px); + border-bottom: 1px solid var(--line); +} +.header-row { + display: flex; align-items: center; gap: 20px; + min-height: 60px; +} +.brand { + display: inline-flex; align-items: center; gap: 10px; + color: var(--ink); font-weight: 660; letter-spacing: -0.02em; +} +.brand:hover { text-decoration: none; } +.mark { + width: 26px; height: 26px; border-radius: 7px; + background: var(--accent); color: var(--accent-ink); + display: grid; place-items: center; + font: 700 14px/1 var(--sans); +} +.site-nav { margin-left: auto; display: flex; align-items: center; gap: 4px; flex-wrap: wrap; } +.site-nav a { + color: var(--ink-2); padding: 7px 11px; border-radius: 7px; + font-size: 0.925rem; +} +.site-nav a:hover { background: var(--surface-2); color: var(--ink); text-decoration: none; } +.site-nav a.active { color: var(--accent); background: var(--accent-soft); } + +/* ---------- sections ---------- */ + +section { padding: 56px 0; border-top: 1px solid var(--line); } +section:first-of-type { border-top: 0; } +.section-head { max-width: 62ch; margin-bottom: 28px; } +.section-head p { color: var(--muted); margin-top: 8px; } +.eyebrow { + display: block; font: 600 0.76rem/1 var(--sans); + letter-spacing: 0.09em; text-transform: uppercase; + color: var(--accent); margin-bottom: 10px; +} + +/* ---------- hero ---------- */ + +.hero { padding: 72px 0 60px; } +.hero h1 { max-width: 20ch; } +.hero .tagline { + font-size: clamp(1.05rem, 2.4vw, 1.3rem); + color: var(--ink-2); margin-top: 16px; max-width: 48ch; +} +.pill { + display: inline-flex; align-items: center; gap: 8px; + font-size: 0.82rem; font-weight: 550; color: var(--ink-2); + background: var(--surface); border: 1px solid var(--line); + border-radius: 999px; padding: 5px 13px; margin-bottom: 22px; + box-shadow: var(--shadow-sm); +} +.pill .dot { + width: 7px; height: 7px; border-radius: 50%; + background: var(--ok); +} +.actions { display: flex; flex-wrap: wrap; gap: 10px; margin-top: 28px; } + +.btn { + display: inline-flex; align-items: center; gap: 8px; + font: 550 0.94rem/1 var(--sans); + padding: 11px 17px; border-radius: var(--radius-sm); + border: 1px solid var(--line-strong); + background: var(--surface); color: var(--ink); + box-shadow: var(--shadow-sm); + cursor: pointer; +} +.btn:hover { text-decoration: none; border-color: var(--muted); } +.btn-primary { + background: var(--accent); border-color: transparent; color: var(--accent-ink); + background-image: linear-gradient(180deg, rgba(255, 255, 255, 0.14), rgba(255, 255, 255, 0)); +} +.btn-primary:hover { filter: brightness(1.06); } + +/* ---------- cards ---------- */ + +.grid { display: grid; gap: 16px; } +.grid-3 { grid-template-columns: repeat(auto-fit, minmax(255px, 1fr)); } + +.card { + background: var(--surface); + border: 1px solid var(--line); + border-radius: var(--radius); + padding: 22px; + box-shadow: var(--shadow); +} +.card h3 { margin-bottom: 8px; } +.card p { color: var(--ink-2); font-size: 0.945rem; } +.card ul { margin: 14px 0 0; padding-left: 18px; color: var(--muted); font-size: 0.9rem; } +.card li + li { margin-top: 5px; } +.step { + display: grid; place-items: center; + width: 25px; height: 25px; border-radius: 7px; + background: var(--accent-soft); color: var(--accent); + border: 1px solid var(--accent-line); + font: 650 0.82rem/1 var(--mono); + margin-bottom: 13px; +} + +.note { + margin-top: 20px; padding: 15px 18px; + background: var(--accent-soft); + border: 1px solid var(--accent-line); + border-radius: var(--radius-sm); + color: var(--ink-2); font-size: 0.93rem; +} + +/* ---------- terminal ---------- */ + +.terminal { + background: var(--term-bg); + border: 1px solid var(--line-strong); + border-radius: var(--radius); + box-shadow: var(--shadow); + overflow: hidden; +} +.term-bar { + display: flex; align-items: center; gap: 9px; + padding: 11px 14px; + border-bottom: 1px solid rgba(255, 255, 255, 0.07); + background: rgba(255, 255, 255, 0.03); +} +.term-bar .lamp { width: 10px; height: 10px; border-radius: 50%; background: #39404e; } +.term-bar .name { + margin-left: 6px; color: var(--term-dim); + font: 500 0.8rem/1 var(--mono); +} +.terminal pre { + margin: 0; padding: 18px; + overflow-x: auto; + color: var(--term-ink); + font-size: 0.815rem; line-height: 1.75; + tab-size: 2; +} +.terminal .t-dim { color: var(--term-dim); } +.terminal .t-ok { color: var(--term-ok); } +.terminal .t-url { color: var(--term-url); } +.terminal .t-redact { color: var(--term-redact); } +.terminal .t-prompt { color: var(--term-ok); } +.term-legend { + margin-top: 14px; color: var(--muted); font-size: 0.88rem; + max-width: 68ch; +} + +/* ---------- code blocks ---------- */ + +pre.code { + margin: 0; + background: var(--surface); + border: 1px solid var(--line); + border-radius: var(--radius); + padding: 18px; + overflow-x: auto; + font-size: 0.85rem; line-height: 1.75; + box-shadow: var(--shadow-sm); + color: var(--ink); +} +pre.code .c { color: var(--muted); } + +/* ---------- table ---------- */ + +.table-wrap { + border: 1px solid var(--line); border-radius: var(--radius); + overflow: hidden; background: var(--surface); box-shadow: var(--shadow-sm); +} +table { width: 100%; border-collapse: collapse; font-size: 0.93rem; } +th, td { text-align: left; padding: 12px 16px; border-bottom: 1px solid var(--line); vertical-align: top; } +th { background: var(--surface-2); font-weight: 600; font-size: 0.8rem; letter-spacing: 0.04em; text-transform: uppercase; color: var(--muted); } +tbody tr:last-child td { border-bottom: 0; } +td code { white-space: nowrap; } +td p { color: var(--ink-2); } + +/* ---------- source / stats ---------- */ + +.stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(120px, 1fr)); gap: 12px; margin-bottom: 20px; } +.stat { + background: var(--surface); border: 1px solid var(--line); + border-radius: var(--radius-sm); padding: 15px 16px; box-shadow: var(--shadow-sm); +} +.stat-value { display: block; font: 650 1.4rem/1.1 var(--sans); letter-spacing: -0.02em; } +.stat-label { display: block; margin-top: 5px; color: var(--muted); font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.06em; } + +.releases { list-style: none; margin: 16px 0 0; padding: 0; } +.releases li { padding: 14px 0; border-top: 1px solid var(--line); } +.release-title { font-weight: 600; } +.release-meta { color: var(--muted); font-size: 0.85rem; margin-top: 3px; } +.release-body { color: var(--ink-2); font-size: 0.9rem; margin-top: 7px; } +.status-msg { color: var(--muted); font-size: 0.92rem; } + +/* ---------- footer ---------- */ + +.site-footer { + border-top: 1px solid var(--line); + padding: 32px 0 48px; + color: var(--muted); font-size: 0.9rem; +} +.footer-row { display: flex; flex-wrap: wrap; gap: 8px 18px; align-items: center; } +.footer-row nav { display: flex; gap: 18px; margin-left: auto; flex-wrap: wrap; } + +/* ---------- docs page ---------- */ + +.docs-layout { display: grid; grid-template-columns: 220px minmax(0, 1fr); gap: 28px; align-items: start; } +.docs-nav { position: sticky; top: 80px; } +.docs-nav h3 { font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.07em; color: var(--muted); margin-bottom: 10px; } +.docs-nav ul { list-style: none; margin: 0; padding: 0; } +.docs-nav li + li { margin-top: 2px; } +.docs-nav a { + display: block; padding: 7px 10px; border-radius: 7px; + color: var(--ink-2); font-size: 0.9rem; +} +.docs-nav a:hover { background: var(--surface-2); color: var(--ink); text-decoration: none; } +.docs-nav a.active { background: var(--accent-soft); color: var(--accent); font-weight: 550; } + +/* min-width:0 so wide code blocks scroll instead of stretching the grid. */ +.docs-main { min-width: 0; } + +.docs-toolbar { + display: flex; align-items: center; gap: 12px; flex-wrap: wrap; + padding-bottom: 12px; margin-bottom: 20px; border-bottom: 1px solid var(--line); +} +.docs-toolbar strong { font-family: var(--mono); font-size: 0.9rem; font-weight: 600; } +.docs-toolbar a { margin-left: auto; font-size: 0.88rem; } + +.docs-content { max-width: 74ch; } +.docs-content > *:first-child { margin-top: 0; } +.docs-content h1 { font-size: 1.85rem; margin: 36px 0 14px; } +.docs-content h2 { font-size: 1.32rem; margin: 34px 0 12px; padding-top: 14px; border-top: 1px solid var(--line); } +.docs-content h3 { font-size: 1.04rem; margin: 24px 0 8px; } +.docs-content p, .docs-content ul, .docs-content ol { margin: 0 0 14px; } +.docs-content ul, .docs-content ol { padding-left: 22px; } +.docs-content li + li { margin-top: 4px; } +.docs-content pre { + background: var(--surface); border: 1px solid var(--line); border-radius: var(--radius-sm); + padding: 15px; overflow-x: auto; font-size: 0.84rem; line-height: 1.7; margin: 0 0 16px; +} +.docs-content pre code { background: none; border: 0; padding: 0; font-size: inherit; } +.docs-content table { margin: 0 0 18px; border: 1px solid var(--line); border-radius: var(--radius-sm); overflow: hidden; } +.docs-content blockquote { + margin: 0 0 16px; padding: 2px 0 2px 16px; + border-left: 3px solid var(--accent-line); color: var(--ink-2); +} +.docs-content hr { border: 0; border-top: 1px solid var(--line); margin: 28px 0; } +.docs-content img { max-width: 100%; } +.docs-content input[type="checkbox"] { margin-right: 6px; } + +/* ---------- telegram invite ---------- */ + +.tg-modal[hidden] { display: none !important; } +.tg-modal { position: fixed; inset: 0; z-index: 60; display: grid; place-items: center; padding: 20px; } +.tg-backdrop { position: absolute; inset: 0; background: rgba(21, 24, 29, 0.4); backdrop-filter: blur(3px); } +.tg-dialog { + position: relative; width: 100%; max-width: 400px; + background: var(--surface); border: 1px solid var(--line-strong); + border-radius: var(--radius); padding: 26px; + box-shadow: 0 20px 60px -14px rgba(16, 20, 30, 0.3); +} +.tg-dialog h2 { font-size: 1.2rem; margin-bottom: 8px; } +.tg-dialog p { color: var(--ink-2); font-size: 0.93rem; } +.tg-actions { display: flex; gap: 10px; margin-top: 22px; flex-wrap: wrap; } +.tg-close { + position: absolute; top: 12px; right: 12px; + width: 30px; height: 30px; border-radius: 7px; + border: 1px solid var(--line); background: var(--surface); + color: var(--muted); font-size: 1rem; line-height: 1; cursor: pointer; +} +.tg-close:hover { color: var(--ink); background: var(--surface-2); } + +/* ---------- responsive ---------- */ + +@media (max-width: 820px) { + .docs-layout { grid-template-columns: 1fr; } + .docs-nav { position: static; } + .docs-nav ul { display: flex; flex-wrap: wrap; gap: 4px; } +} + +@media (max-width: 640px) { + .shell { padding: 0 16px; } + section { padding: 44px 0; } + .hero { padding: 48px 0 40px; } + .header-row { min-height: 54px; gap: 10px; } + .site-nav { gap: 0; } + .site-nav a { padding: 6px 8px; font-size: 0.875rem; } + .card { padding: 18px; } + .terminal pre { font-size: 0.74rem; padding: 14px; } + .footer-row nav { margin-left: 0; } +} diff --git a/www/vendor/marked.min.js b/www/vendor/marked.min.js new file mode 100644 index 0000000..4052d1b --- /dev/null +++ b/www/vendor/marked.min.js @@ -0,0 +1,6 @@ +/** + * marked v15.0.7 - a markdown parser + * Copyright (c) 2011-2025, Christopher Jeffrey. (MIT Licensed) + * https://github.com/markedjs/marked + */ +!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?t(exports):"function"==typeof define&&define.amd?define(["exports"],t):t((e="undefined"!=typeof globalThis?globalThis:e||self).marked={})}(this,(function(e){"use strict";function t(){return{async:!1,breaks:!1,extensions:null,gfm:!0,hooks:null,pedantic:!1,renderer:null,silent:!1,tokenizer:null,walkTokens:null}}function n(t){e.defaults=t}e.defaults={async:!1,breaks:!1,extensions:null,gfm:!0,hooks:null,pedantic:!1,renderer:null,silent:!1,tokenizer:null,walkTokens:null};const s={exec:()=>null};function r(e,t=""){let n="string"==typeof e?e:e.source;const s={replace:(e,t)=>{let r="string"==typeof t?t:t.source;return r=r.replace(i.caret,"$1"),n=n.replace(e,r),s},getRegex:()=>new RegExp(n,t)};return s}const i={codeRemoveIndent:/^(?: {1,4}| {0,3}\t)/gm,outputLinkReplace:/\\([\[\]])/g,indentCodeCompensation:/^(\s+)(?:```)/,beginningSpace:/^\s+/,endingHash:/#$/,startingSpaceChar:/^ /,endingSpaceChar:/ $/,nonSpaceChar:/[^ ]/,newLineCharGlobal:/\n/g,tabCharGlobal:/\t/g,multipleSpaceGlobal:/\s+/g,blankLine:/^[ \t]*$/,doubleBlankLine:/\n[ \t]*\n[ \t]*$/,blockquoteStart:/^ {0,3}>/,blockquoteSetextReplace:/\n {0,3}((?:=+|-+) *)(?=\n|$)/g,blockquoteSetextReplace2:/^ {0,3}>[ \t]?/gm,listReplaceTabs:/^\t+/,listReplaceNesting:/^ {1,4}(?=( {4})*[^ ])/g,listIsTask:/^\[[ xX]\] /,listReplaceTask:/^\[[ xX]\] +/,anyLine:/\n.*\n/,hrefBrackets:/^<(.*)>$/,tableDelimiter:/[:|]/,tableAlignChars:/^\||\| *$/g,tableRowBlankLine:/\n[ \t]*$/,tableAlignRight:/^ *-+: *$/,tableAlignCenter:/^ *:-+: *$/,tableAlignLeft:/^ *:-+ *$/,startATag:/^/i,startPreScriptTag:/^<(pre|code|kbd|script)(\s|>)/i,endPreScriptTag:/^<\/(pre|code|kbd|script)(\s|>)/i,startAngleBracket:/^$/,pedanticHrefTitle:/^([^'"]*[^\s])\s+(['"])(.*)\2/,unicodeAlphaNumeric:/[\p{L}\p{N}]/u,escapeTest:/[&<>"']/,escapeReplace:/[&<>"']/g,escapeTestNoEncode:/[<>"']|&(?!(#\d{1,7}|#[Xx][a-fA-F0-9]{1,6}|\w+);)/,escapeReplaceNoEncode:/[<>"']|&(?!(#\d{1,7}|#[Xx][a-fA-F0-9]{1,6}|\w+);)/g,unescapeTest:/&(#(?:\d+)|(?:#x[0-9A-Fa-f]+)|(?:\w+));?/gi,caret:/(^|[^\[])\^/g,percentDecode:/%25/g,findPipe:/\|/g,splitPipe:/ \|/,slashPipe:/\\\|/g,carriageReturn:/\r\n|\r/g,spaceLine:/^ +$/gm,notSpaceStart:/^\S*/,endingNewline:/\n$/,listItemRegex:e=>new RegExp(`^( {0,3}${e})((?:[\t ][^\\n]*)?(?:\\n|$))`),nextBulletRegex:e=>new RegExp(`^ {0,${Math.min(3,e-1)}}(?:[*+-]|\\d{1,9}[.)])((?:[ \t][^\\n]*)?(?:\\n|$))`),hrRegex:e=>new RegExp(`^ {0,${Math.min(3,e-1)}}((?:- *){3,}|(?:_ *){3,}|(?:\\* *){3,})(?:\\n+|$)`),fencesBeginRegex:e=>new RegExp(`^ {0,${Math.min(3,e-1)}}(?:\`\`\`|~~~)`),headingBeginRegex:e=>new RegExp(`^ {0,${Math.min(3,e-1)}}#`),htmlBeginRegex:e=>new RegExp(`^ {0,${Math.min(3,e-1)}}<(?:[a-z].*>|!--)`,"i")},l=/^ {0,3}((?:-[\t ]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})(?:\n+|$)/,o=/(?:[*+-]|\d{1,9}[.)])/,a=/^(?!bull |blockCode|fences|blockquote|heading|html|table)((?:.|\n(?!\s*?\n|bull |blockCode|fences|blockquote|heading|html|table))+?)\n {0,3}(=+|-+) *(?:\n+|$)/,c=r(a).replace(/bull/g,o).replace(/blockCode/g,/(?: {4}| {0,3}\t)/).replace(/fences/g,/ {0,3}(?:`{3,}|~{3,})/).replace(/blockquote/g,/ {0,3}>/).replace(/heading/g,/ {0,3}#{1,6}/).replace(/html/g,/ {0,3}<[^\n>]+>\n/).replace(/\|table/g,"").getRegex(),h=r(a).replace(/bull/g,o).replace(/blockCode/g,/(?: {4}| {0,3}\t)/).replace(/fences/g,/ {0,3}(?:`{3,}|~{3,})/).replace(/blockquote/g,/ {0,3}>/).replace(/heading/g,/ {0,3}#{1,6}/).replace(/html/g,/ {0,3}<[^\n>]+>\n/).replace(/table/g,/ {0,3}\|?(?:[:\- ]*\|)+[\:\- ]*\n/).getRegex(),p=/^([^\n]+(?:\n(?!hr|heading|lheading|blockquote|fences|list|html|table| +\n)[^\n]+)*)/,u=/(?!\s*\])(?:\\.|[^\[\]\\])+/,g=r(/^ {0,3}\[(label)\]: *(?:\n[ \t]*)?([^<\s][^\s]*|<.*?>)(?:(?: +(?:\n[ \t]*)?| *\n[ \t]*)(title))? *(?:\n+|$)/).replace("label",u).replace("title",/(?:"(?:\\"?|[^"\\])*"|'[^'\n]*(?:\n[^'\n]+)*\n?'|\([^()]*\))/).getRegex(),k=r(/^( {0,3}bull)([ \t][^\n]+?)?(?:\n|$)/).replace(/bull/g,o).getRegex(),d="address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|meta|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul",f=/|$))/,x=r("^ {0,3}(?:<(script|pre|style|textarea)[\\s>][\\s\\S]*?(?:[^\\n]*\\n+|$)|comment[^\\n]*(\\n+|$)|<\\?[\\s\\S]*?(?:\\?>\\n*|$)|\\n*|$)|\\n*|$)|)[\\s\\S]*?(?:(?:\\n[ \t]*)+\\n|$)|<(?!script|pre|style|textarea)([a-z][\\w-]*)(?:attribute)*? */?>(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n[ \t]*)+\\n|$)|(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n[ \t]*)+\\n|$))","i").replace("comment",f).replace("tag",d).replace("attribute",/ +[a-zA-Z:_][\w.:-]*(?: *= *"[^"\n]*"| *= *'[^'\n]*'| *= *[^\s"'=<>`]+)?/).getRegex(),b=r(p).replace("hr",l).replace("heading"," {0,3}#{1,6}(?:\\s|$)").replace("|lheading","").replace("|table","").replace("blockquote"," {0,3}>").replace("fences"," {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n").replace("list"," {0,3}(?:[*+-]|1[.)]) ").replace("html",")|<(?:script|pre|style|textarea|!--)").replace("tag",d).getRegex(),w={blockquote:r(/^( {0,3}> ?(paragraph|[^\n]*)(?:\n|$))+/).replace("paragraph",b).getRegex(),code:/^((?: {4}| {0,3}\t)[^\n]+(?:\n(?:[ \t]*(?:\n|$))*)?)+/,def:g,fences:/^ {0,3}(`{3,}(?=[^`\n]*(?:\n|$))|~{3,})([^\n]*)(?:\n|$)(?:|([\s\S]*?)(?:\n|$))(?: {0,3}\1[~`]* *(?=\n|$)|$)/,heading:/^ {0,3}(#{1,6})(?=\s|$)(.*)(?:\n+|$)/,hr:l,html:x,lheading:c,list:k,newline:/^(?:[ \t]*(?:\n|$))+/,paragraph:b,table:s,text:/^[^\n]+/},m=r("^ *([^\\n ].*)\\n {0,3}((?:\\| *)?:?-+:? *(?:\\| *:?-+:? *)*(?:\\| *)?)(?:\\n((?:(?! *\\n|hr|heading|blockquote|code|fences|list|html).*(?:\\n|$))*)\\n*|$)").replace("hr",l).replace("heading"," {0,3}#{1,6}(?:\\s|$)").replace("blockquote"," {0,3}>").replace("code","(?: {4}| {0,3}\t)[^\\n]").replace("fences"," {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n").replace("list"," {0,3}(?:[*+-]|1[.)]) ").replace("html",")|<(?:script|pre|style|textarea|!--)").replace("tag",d).getRegex(),y={...w,lheading:h,table:m,paragraph:r(p).replace("hr",l).replace("heading"," {0,3}#{1,6}(?:\\s|$)").replace("|lheading","").replace("table",m).replace("blockquote"," {0,3}>").replace("fences"," {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n").replace("list"," {0,3}(?:[*+-]|1[.)]) ").replace("html",")|<(?:script|pre|style|textarea|!--)").replace("tag",d).getRegex()},$={...w,html:r("^ *(?:comment *(?:\\n|\\s*$)|<(tag)[\\s\\S]+? *(?:\\n{2,}|\\s*$)|\\s]*)*?/?> *(?:\\n{2,}|\\s*$))").replace("comment",f).replace(/tag/g,"(?!(?:a|em|strong|small|s|cite|q|dfn|abbr|data|time|code|var|samp|kbd|sub|sup|i|b|u|mark|ruby|rt|rp|bdi|bdo|span|br|wbr|ins|del|img)\\b)\\w+(?!:|[^\\w\\s@]*@)\\b").getRegex(),def:/^ *\[([^\]]+)\]: *]+)>?(?: +(["(][^\n]+[")]))? *(?:\n+|$)/,heading:/^(#{1,6})(.*)(?:\n+|$)/,fences:s,lheading:/^(.+?)\n {0,3}(=+|-+) *(?:\n+|$)/,paragraph:r(p).replace("hr",l).replace("heading"," *#{1,6} *[^\n]").replace("lheading",c).replace("|table","").replace("blockquote"," {0,3}>").replace("|fences","").replace("|list","").replace("|html","").replace("|tag","").getRegex()},R=/^( {2,}|\\)\n(?!\s*$)/,S=/[\p{P}\p{S}]/u,T=/[\s\p{P}\p{S}]/u,z=/[^\s\p{P}\p{S}]/u,A=r(/^((?![*_])punctSpace)/,"u").replace(/punctSpace/g,T).getRegex(),_=/(?!~)[\p{P}\p{S}]/u,P=/^(?:\*+(?:((?!\*)punct)|[^\s*]))|^_+(?:((?!_)punct)|([^\s_]))/,I=r(P,"u").replace(/punct/g,S).getRegex(),L=r(P,"u").replace(/punct/g,_).getRegex(),B="^[^_*]*?__[^_*]*?\\*[^_*]*?(?=__)|[^*]+(?=[^*])|(?!\\*)punct(\\*+)(?=[\\s]|$)|notPunctSpace(\\*+)(?!\\*)(?=punctSpace|$)|(?!\\*)punctSpace(\\*+)(?=notPunctSpace)|[\\s](\\*+)(?!\\*)(?=punct)|(?!\\*)punct(\\*+)(?!\\*)(?=punct)|notPunctSpace(\\*+)(?=notPunctSpace)",C=r(B,"gu").replace(/notPunctSpace/g,z).replace(/punctSpace/g,T).replace(/punct/g,S).getRegex(),q=r(B,"gu").replace(/notPunctSpace/g,/(?:[^\s\p{P}\p{S}]|~)/u).replace(/punctSpace/g,/(?!~)[\s\p{P}\p{S}]/u).replace(/punct/g,_).getRegex(),E=r("^[^_*]*?\\*\\*[^_*]*?_[^_*]*?(?=\\*\\*)|[^_]+(?=[^_])|(?!_)punct(_+)(?=[\\s]|$)|notPunctSpace(_+)(?!_)(?=punctSpace|$)|(?!_)punctSpace(_+)(?=notPunctSpace)|[\\s](_+)(?!_)(?=punct)|(?!_)punct(_+)(?!_)(?=punct)","gu").replace(/notPunctSpace/g,z).replace(/punctSpace/g,T).replace(/punct/g,S).getRegex(),Z=r(/\\(punct)/,"gu").replace(/punct/g,S).getRegex(),v=r(/^<(scheme:[^\s\x00-\x1f<>]*|email)>/).replace("scheme",/[a-zA-Z][a-zA-Z0-9+.-]{1,31}/).replace("email",/[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+(@)[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+(?![-_])/).getRegex(),D=r(f).replace("(?:--\x3e|$)","--\x3e").getRegex(),M=r("^comment|^|^<[a-zA-Z][\\w-]*(?:attribute)*?\\s*/?>|^<\\?[\\s\\S]*?\\?>|^|^").replace("comment",D).replace("attribute",/\s+[a-zA-Z:_][\w.:-]*(?:\s*=\s*"[^"]*"|\s*=\s*'[^']*'|\s*=\s*[^\s"'=<>`]+)?/).getRegex(),O=/(?:\[(?:\\.|[^\[\]\\])*\]|\\.|`[^`]*`|[^\[\]\\`])*?/,Q=r(/^!?\[(label)\]\(\s*(href)(?:\s+(title))?\s*\)/).replace("label",O).replace("href",/<(?:\\.|[^\n<>\\])+>|[^\s\x00-\x1f]*/).replace("title",/"(?:\\"?|[^"\\])*"|'(?:\\'?|[^'\\])*'|\((?:\\\)?|[^)\\])*\)/).getRegex(),j=r(/^!?\[(label)\]\[(ref)\]/).replace("label",O).replace("ref",u).getRegex(),N=r(/^!?\[(ref)\](?:\[\])?/).replace("ref",u).getRegex(),G={_backpedal:s,anyPunctuation:Z,autolink:v,blockSkip:/\[[^[\]]*?\]\((?:\\.|[^\\\(\)]|\((?:\\.|[^\\\(\)])*\))*\)|`[^`]*?`|<[^<>]*?>/g,br:R,code:/^(`+)([^`]|[^`][\s\S]*?[^`])\1(?!`)/,del:s,emStrongLDelim:I,emStrongRDelimAst:C,emStrongRDelimUnd:E,escape:/^\\([!"#$%&'()*+,\-./:;<=>?@\[\]\\^_`{|}~])/,link:Q,nolink:N,punctuation:A,reflink:j,reflinkSearch:r("reflink|nolink(?!\\()","g").replace("reflink",j).replace("nolink",N).getRegex(),tag:M,text:/^(`+|[^`])(?:(?= {2,}\n)|[\s\S]*?(?:(?=[\\":">",'"':""","'":"'"},V=e=>K[e];function W(e,t){if(t){if(i.escapeTest.test(e))return e.replace(i.escapeReplace,V)}else if(i.escapeTestNoEncode.test(e))return e.replace(i.escapeReplaceNoEncode,V);return e}function Y(e){try{e=encodeURI(e).replace(i.percentDecode,"%")}catch{return null}return e}function ee(e,t){const n=e.replace(i.findPipe,((e,t,n)=>{let s=!1,r=t;for(;--r>=0&&"\\"===n[r];)s=!s;return s?"|":" |"})).split(i.splitPipe);let s=0;if(n[0].trim()||n.shift(),n.length>0&&!n.at(-1)?.trim()&&n.pop(),t)if(n.length>t)n.splice(t);else for(;n.length0)return{type:"space",raw:t[0]}}code(e){const t=this.rules.block.code.exec(e);if(t){const e=t[0].replace(this.rules.other.codeRemoveIndent,"");return{type:"code",raw:t[0],codeBlockStyle:"indented",text:this.options.pedantic?e:te(e,"\n")}}}fences(e){const t=this.rules.block.fences.exec(e);if(t){const e=t[0],n=function(e,t,n){const s=e.match(n.other.indentCodeCompensation);if(null===s)return t;const r=s[1];return t.split("\n").map((e=>{const t=e.match(n.other.beginningSpace);if(null===t)return e;const[s]=t;return s.length>=r.length?e.slice(r.length):e})).join("\n")}(e,t[3]||"",this.rules);return{type:"code",raw:e,lang:t[2]?t[2].trim().replace(this.rules.inline.anyPunctuation,"$1"):t[2],text:n}}}heading(e){const t=this.rules.block.heading.exec(e);if(t){let e=t[2].trim();if(this.rules.other.endingHash.test(e)){const t=te(e,"#");this.options.pedantic?e=t.trim():t&&!this.rules.other.endingSpaceChar.test(t)||(e=t.trim())}return{type:"heading",raw:t[0],depth:t[1].length,text:e,tokens:this.lexer.inline(e)}}}hr(e){const t=this.rules.block.hr.exec(e);if(t)return{type:"hr",raw:te(t[0],"\n")}}blockquote(e){const t=this.rules.block.blockquote.exec(e);if(t){let e=te(t[0],"\n").split("\n"),n="",s="";const r=[];for(;e.length>0;){let t=!1;const i=[];let l;for(l=0;l1,r={type:"list",raw:"",ordered:s,start:s?+n.slice(0,-1):"",loose:!1,items:[]};n=s?`\\d{1,9}\\${n.slice(-1)}`:`\\${n}`,this.options.pedantic&&(n=s?n:"[*+-]");const i=this.rules.other.listItemRegex(n);let l=!1;for(;e;){let n=!1,s="",o="";if(!(t=i.exec(e)))break;if(this.rules.block.hr.test(e))break;s=t[0],e=e.substring(s.length);let a=t[2].split("\n",1)[0].replace(this.rules.other.listReplaceTabs,(e=>" ".repeat(3*e.length))),c=e.split("\n",1)[0],h=!a.trim(),p=0;if(this.options.pedantic?(p=2,o=a.trimStart()):h?p=t[1].length+1:(p=t[2].search(this.rules.other.nonSpaceChar),p=p>4?1:p,o=a.slice(p),p+=t[1].length),h&&this.rules.other.blankLine.test(c)&&(s+=c+"\n",e=e.substring(c.length+1),n=!0),!n){const t=this.rules.other.nextBulletRegex(p),n=this.rules.other.hrRegex(p),r=this.rules.other.fencesBeginRegex(p),i=this.rules.other.headingBeginRegex(p),l=this.rules.other.htmlBeginRegex(p);for(;e;){const u=e.split("\n",1)[0];let g;if(c=u,this.options.pedantic?(c=c.replace(this.rules.other.listReplaceNesting," "),g=c):g=c.replace(this.rules.other.tabCharGlobal," "),r.test(c))break;if(i.test(c))break;if(l.test(c))break;if(t.test(c))break;if(n.test(c))break;if(g.search(this.rules.other.nonSpaceChar)>=p||!c.trim())o+="\n"+g.slice(p);else{if(h)break;if(a.replace(this.rules.other.tabCharGlobal," ").search(this.rules.other.nonSpaceChar)>=4)break;if(r.test(a))break;if(i.test(a))break;if(n.test(a))break;o+="\n"+c}h||c.trim()||(h=!0),s+=u+"\n",e=e.substring(u.length+1),a=g.slice(p)}}r.loose||(l?r.loose=!0:this.rules.other.doubleBlankLine.test(s)&&(l=!0));let u,g=null;this.options.gfm&&(g=this.rules.other.listIsTask.exec(o),g&&(u="[ ] "!==g[0],o=o.replace(this.rules.other.listReplaceTask,""))),r.items.push({type:"list_item",raw:s,task:!!g,checked:u,loose:!1,text:o,tokens:[]}),r.raw+=s}const o=r.items.at(-1);if(!o)return;o.raw=o.raw.trimEnd(),o.text=o.text.trimEnd(),r.raw=r.raw.trimEnd();for(let e=0;e"space"===e.type)),n=t.length>0&&t.some((e=>this.rules.other.anyLine.test(e.raw)));r.loose=n}if(r.loose)for(let e=0;e({text:e,tokens:this.lexer.inline(e),header:!1,align:i.align[t]}))));return i}}lheading(e){const t=this.rules.block.lheading.exec(e);if(t)return{type:"heading",raw:t[0],depth:"="===t[2].charAt(0)?1:2,text:t[1],tokens:this.lexer.inline(t[1])}}paragraph(e){const t=this.rules.block.paragraph.exec(e);if(t){const e="\n"===t[1].charAt(t[1].length-1)?t[1].slice(0,-1):t[1];return{type:"paragraph",raw:t[0],text:e,tokens:this.lexer.inline(e)}}}text(e){const t=this.rules.block.text.exec(e);if(t)return{type:"text",raw:t[0],text:t[0],tokens:this.lexer.inline(t[0])}}escape(e){const t=this.rules.inline.escape.exec(e);if(t)return{type:"escape",raw:t[0],text:t[1]}}tag(e){const t=this.rules.inline.tag.exec(e);if(t)return!this.lexer.state.inLink&&this.rules.other.startATag.test(t[0])?this.lexer.state.inLink=!0:this.lexer.state.inLink&&this.rules.other.endATag.test(t[0])&&(this.lexer.state.inLink=!1),!this.lexer.state.inRawBlock&&this.rules.other.startPreScriptTag.test(t[0])?this.lexer.state.inRawBlock=!0:this.lexer.state.inRawBlock&&this.rules.other.endPreScriptTag.test(t[0])&&(this.lexer.state.inRawBlock=!1),{type:"html",raw:t[0],inLink:this.lexer.state.inLink,inRawBlock:this.lexer.state.inRawBlock,block:!1,text:t[0]}}link(e){const t=this.rules.inline.link.exec(e);if(t){const e=t[2].trim();if(!this.options.pedantic&&this.rules.other.startAngleBracket.test(e)){if(!this.rules.other.endAngleBracket.test(e))return;const t=te(e.slice(0,-1),"\\");if((e.length-t.length)%2==0)return}else{const e=function(e,t){if(-1===e.indexOf(t[1]))return-1;let n=0;for(let s=0;s-1){const n=(0===t[0].indexOf("!")?5:4)+t[1].length+e;t[2]=t[2].substring(0,e),t[0]=t[0].substring(0,n).trim(),t[3]=""}}let n=t[2],s="";if(this.options.pedantic){const e=this.rules.other.pedanticHrefTitle.exec(n);e&&(n=e[1],s=e[3])}else s=t[3]?t[3].slice(1,-1):"";return n=n.trim(),this.rules.other.startAngleBracket.test(n)&&(n=this.options.pedantic&&!this.rules.other.endAngleBracket.test(e)?n.slice(1):n.slice(1,-1)),ne(t,{href:n?n.replace(this.rules.inline.anyPunctuation,"$1"):n,title:s?s.replace(this.rules.inline.anyPunctuation,"$1"):s},t[0],this.lexer,this.rules)}}reflink(e,t){let n;if((n=this.rules.inline.reflink.exec(e))||(n=this.rules.inline.nolink.exec(e))){const e=t[(n[2]||n[1]).replace(this.rules.other.multipleSpaceGlobal," ").toLowerCase()];if(!e){const e=n[0].charAt(0);return{type:"text",raw:e,text:e}}return ne(n,e,n[0],this.lexer,this.rules)}}emStrong(e,t,n=""){let s=this.rules.inline.emStrongLDelim.exec(e);if(!s)return;if(s[3]&&n.match(this.rules.other.unicodeAlphaNumeric))return;if(!(s[1]||s[2]||"")||!n||this.rules.inline.punctuation.exec(n)){const n=[...s[0]].length-1;let r,i,l=n,o=0;const a="*"===s[0][0]?this.rules.inline.emStrongRDelimAst:this.rules.inline.emStrongRDelimUnd;for(a.lastIndex=0,t=t.slice(-1*e.length+n);null!=(s=a.exec(t));){if(r=s[1]||s[2]||s[3]||s[4]||s[5]||s[6],!r)continue;if(i=[...r].length,s[3]||s[4]){l+=i;continue}if((s[5]||s[6])&&n%3&&!((n+i)%3)){o+=i;continue}if(l-=i,l>0)continue;i=Math.min(i,i+l+o);const t=[...s[0]][0].length,a=e.slice(0,n+s.index+t+i);if(Math.min(n,i)%2){const e=a.slice(1,-1);return{type:"em",raw:a,text:e,tokens:this.lexer.inlineTokens(e)}}const c=a.slice(2,-2);return{type:"strong",raw:a,text:c,tokens:this.lexer.inlineTokens(c)}}}}codespan(e){const t=this.rules.inline.code.exec(e);if(t){let e=t[2].replace(this.rules.other.newLineCharGlobal," ");const n=this.rules.other.nonSpaceChar.test(e),s=this.rules.other.startingSpaceChar.test(e)&&this.rules.other.endingSpaceChar.test(e);return n&&s&&(e=e.substring(1,e.length-1)),{type:"codespan",raw:t[0],text:e}}}br(e){const t=this.rules.inline.br.exec(e);if(t)return{type:"br",raw:t[0]}}del(e){const t=this.rules.inline.del.exec(e);if(t)return{type:"del",raw:t[0],text:t[2],tokens:this.lexer.inlineTokens(t[2])}}autolink(e){const t=this.rules.inline.autolink.exec(e);if(t){let e,n;return"@"===t[2]?(e=t[1],n="mailto:"+e):(e=t[1],n=e),{type:"link",raw:t[0],text:e,href:n,tokens:[{type:"text",raw:e,text:e}]}}}url(e){let t;if(t=this.rules.inline.url.exec(e)){let e,n;if("@"===t[2])e=t[0],n="mailto:"+e;else{let s;do{s=t[0],t[0]=this.rules.inline._backpedal.exec(t[0])?.[0]??""}while(s!==t[0]);e=t[0],n="www."===t[1]?"http://"+t[0]:t[0]}return{type:"link",raw:t[0],text:e,href:n,tokens:[{type:"text",raw:e,text:e}]}}}inlineText(e){const t=this.rules.inline.text.exec(e);if(t){const e=this.lexer.state.inRawBlock;return{type:"text",raw:t[0],text:t[0],escaped:e}}}}class re{tokens;options;state;tokenizer;inlineQueue;constructor(t){this.tokens=[],this.tokens.links=Object.create(null),this.options=t||e.defaults,this.options.tokenizer=this.options.tokenizer||new se,this.tokenizer=this.options.tokenizer,this.tokenizer.options=this.options,this.tokenizer.lexer=this,this.inlineQueue=[],this.state={inLink:!1,inRawBlock:!1,top:!0};const n={other:i,block:U.normal,inline:J.normal};this.options.pedantic?(n.block=U.pedantic,n.inline=J.pedantic):this.options.gfm&&(n.block=U.gfm,this.options.breaks?n.inline=J.breaks:n.inline=J.gfm),this.tokenizer.rules=n}static get rules(){return{block:U,inline:J}}static lex(e,t){return new re(t).lex(e)}static lexInline(e,t){return new re(t).inlineTokens(e)}lex(e){e=e.replace(i.carriageReturn,"\n"),this.blockTokens(e,this.tokens);for(let e=0;e!!(s=n.call({lexer:this},e,t))&&(e=e.substring(s.raw.length),t.push(s),!0))))continue;if(s=this.tokenizer.space(e)){e=e.substring(s.raw.length);const n=t.at(-1);1===s.raw.length&&void 0!==n?n.raw+="\n":t.push(s);continue}if(s=this.tokenizer.code(e)){e=e.substring(s.raw.length);const n=t.at(-1);"paragraph"===n?.type||"text"===n?.type?(n.raw+="\n"+s.raw,n.text+="\n"+s.text,this.inlineQueue.at(-1).src=n.text):t.push(s);continue}if(s=this.tokenizer.fences(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.heading(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.hr(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.blockquote(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.list(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.html(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.def(e)){e=e.substring(s.raw.length);const n=t.at(-1);"paragraph"===n?.type||"text"===n?.type?(n.raw+="\n"+s.raw,n.text+="\n"+s.raw,this.inlineQueue.at(-1).src=n.text):this.tokens.links[s.tag]||(this.tokens.links[s.tag]={href:s.href,title:s.title});continue}if(s=this.tokenizer.table(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.lheading(e)){e=e.substring(s.raw.length),t.push(s);continue}let r=e;if(this.options.extensions?.startBlock){let t=1/0;const n=e.slice(1);let s;this.options.extensions.startBlock.forEach((e=>{s=e.call({lexer:this},n),"number"==typeof s&&s>=0&&(t=Math.min(t,s))})),t<1/0&&t>=0&&(r=e.substring(0,t+1))}if(this.state.top&&(s=this.tokenizer.paragraph(r))){const i=t.at(-1);n&&"paragraph"===i?.type?(i.raw+="\n"+s.raw,i.text+="\n"+s.text,this.inlineQueue.pop(),this.inlineQueue.at(-1).src=i.text):t.push(s),n=r.length!==e.length,e=e.substring(s.raw.length)}else if(s=this.tokenizer.text(e)){e=e.substring(s.raw.length);const n=t.at(-1);"text"===n?.type?(n.raw+="\n"+s.raw,n.text+="\n"+s.text,this.inlineQueue.pop(),this.inlineQueue.at(-1).src=n.text):t.push(s)}else if(e){const t="Infinite loop on byte: "+e.charCodeAt(0);if(this.options.silent){console.error(t);break}throw new Error(t)}}return this.state.top=!0,t}inline(e,t=[]){return this.inlineQueue.push({src:e,tokens:t}),t}inlineTokens(e,t=[]){let n=e,s=null;if(this.tokens.links){const e=Object.keys(this.tokens.links);if(e.length>0)for(;null!=(s=this.tokenizer.rules.inline.reflinkSearch.exec(n));)e.includes(s[0].slice(s[0].lastIndexOf("[")+1,-1))&&(n=n.slice(0,s.index)+"["+"a".repeat(s[0].length-2)+"]"+n.slice(this.tokenizer.rules.inline.reflinkSearch.lastIndex))}for(;null!=(s=this.tokenizer.rules.inline.blockSkip.exec(n));)n=n.slice(0,s.index)+"["+"a".repeat(s[0].length-2)+"]"+n.slice(this.tokenizer.rules.inline.blockSkip.lastIndex);for(;null!=(s=this.tokenizer.rules.inline.anyPunctuation.exec(n));)n=n.slice(0,s.index)+"++"+n.slice(this.tokenizer.rules.inline.anyPunctuation.lastIndex);let r=!1,i="";for(;e;){let s;if(r||(i=""),r=!1,this.options.extensions?.inline?.some((n=>!!(s=n.call({lexer:this},e,t))&&(e=e.substring(s.raw.length),t.push(s),!0))))continue;if(s=this.tokenizer.escape(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.tag(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.link(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.reflink(e,this.tokens.links)){e=e.substring(s.raw.length);const n=t.at(-1);"text"===s.type&&"text"===n?.type?(n.raw+=s.raw,n.text+=s.text):t.push(s);continue}if(s=this.tokenizer.emStrong(e,n,i)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.codespan(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.br(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.del(e)){e=e.substring(s.raw.length),t.push(s);continue}if(s=this.tokenizer.autolink(e)){e=e.substring(s.raw.length),t.push(s);continue}if(!this.state.inLink&&(s=this.tokenizer.url(e))){e=e.substring(s.raw.length),t.push(s);continue}let l=e;if(this.options.extensions?.startInline){let t=1/0;const n=e.slice(1);let s;this.options.extensions.startInline.forEach((e=>{s=e.call({lexer:this},n),"number"==typeof s&&s>=0&&(t=Math.min(t,s))})),t<1/0&&t>=0&&(l=e.substring(0,t+1))}if(s=this.tokenizer.inlineText(l)){e=e.substring(s.raw.length),"_"!==s.raw.slice(-1)&&(i=s.raw.slice(-1)),r=!0;const n=t.at(-1);"text"===n?.type?(n.raw+=s.raw,n.text+=s.text):t.push(s)}else if(e){const t="Infinite loop on byte: "+e.charCodeAt(0);if(this.options.silent){console.error(t);break}throw new Error(t)}}return t}}class ie{options;parser;constructor(t){this.options=t||e.defaults}space(e){return""}code({text:e,lang:t,escaped:n}){const s=(t||"").match(i.notSpaceStart)?.[0],r=e.replace(i.endingNewline,"")+"\n";return s?'
    '+(n?r:W(r,!0))+"
    \n":"
    "+(n?r:W(r,!0))+"
    \n"}blockquote({tokens:e}){return`
    \n${this.parser.parse(e)}
    \n`}html({text:e}){return e}heading({tokens:e,depth:t}){return`${this.parser.parseInline(e)}\n`}hr(e){return"
    \n"}list(e){const t=e.ordered,n=e.start;let s="";for(let t=0;t\n"+s+"\n"}listitem(e){let t="";if(e.task){const n=this.checkbox({checked:!!e.checked});e.loose?"paragraph"===e.tokens[0]?.type?(e.tokens[0].text=n+" "+e.tokens[0].text,e.tokens[0].tokens&&e.tokens[0].tokens.length>0&&"text"===e.tokens[0].tokens[0].type&&(e.tokens[0].tokens[0].text=n+" "+W(e.tokens[0].tokens[0].text),e.tokens[0].tokens[0].escaped=!0)):e.tokens.unshift({type:"text",raw:n+" ",text:n+" ",escaped:!0}):t+=n+" "}return t+=this.parser.parse(e.tokens,!!e.loose),`
  • ${t}
    • \n`}checkbox({checked:e}){return"'}paragraph({tokens:e}){return`

    ${this.parser.parseInline(e)}

    \n`}table(e){let t="",n="";for(let t=0;t${s}`),"\n\n"+t+"\n"+s+"
    \n"}tablerow({text:e}){return`\n${e}\n`}tablecell(e){const t=this.parser.parseInline(e.tokens),n=e.header?"th":"td";return(e.align?`<${n} align="${e.align}">`:`<${n}>`)+t+`\n`}strong({tokens:e}){return`${this.parser.parseInline(e)}`}em({tokens:e}){return`${this.parser.parseInline(e)}`}codespan({text:e}){return`${W(e,!0)}`}br(e){return"
    "}del({tokens:e}){return`${this.parser.parseInline(e)}`}link({href:e,title:t,tokens:n}){const s=this.parser.parseInline(n),r=Y(e);if(null===r)return s;let i='    ",i}image({href:e,title:t,text:n}){const s=Y(e);if(null===s)return W(n);let r=`${n}{const r=e[s].flat(1/0);n=n.concat(this.walkTokens(r,t))})):e.tokens&&(n=n.concat(this.walkTokens(e.tokens,t)))}}return n}use(...e){const t=this.defaults.extensions||{renderers:{},childTokens:{}};return e.forEach((e=>{const n={...e};if(n.async=this.defaults.async||n.async||!1,e.extensions&&(e.extensions.forEach((e=>{if(!e.name)throw new Error("extension name required");if("renderer"in e){const n=t.renderers[e.name];t.renderers[e.name]=n?function(...t){let s=e.renderer.apply(this,t);return!1===s&&(s=n.apply(this,t)),s}:e.renderer}if("tokenizer"in e){if(!e.level||"block"!==e.level&&"inline"!==e.level)throw new Error("extension level must be 'block' or 'inline'");const n=t[e.level];n?n.unshift(e.tokenizer):t[e.level]=[e.tokenizer],e.start&&("block"===e.level?t.startBlock?t.startBlock.push(e.start):t.startBlock=[e.start]:"inline"===e.level&&(t.startInline?t.startInline.push(e.start):t.startInline=[e.start]))}"childTokens"in e&&e.childTokens&&(t.childTokens[e.name]=e.childTokens)})),n.extensions=t),e.renderer){const t=this.defaults.renderer||new ie(this.defaults);for(const n in e.renderer){if(!(n in t))throw new Error(`renderer '${n}' does not exist`);if(["options","parser"].includes(n))continue;const s=n,r=e.renderer[s],i=t[s];t[s]=(...e)=>{let n=r.apply(t,e);return!1===n&&(n=i.apply(t,e)),n||""}}n.renderer=t}if(e.tokenizer){const t=this.defaults.tokenizer||new se(this.defaults);for(const n in e.tokenizer){if(!(n in t))throw new Error(`tokenizer '${n}' does not exist`);if(["options","rules","lexer"].includes(n))continue;const s=n,r=e.tokenizer[s],i=t[s];t[s]=(...e)=>{let n=r.apply(t,e);return!1===n&&(n=i.apply(t,e)),n}}n.tokenizer=t}if(e.hooks){const t=this.defaults.hooks||new ae;for(const n in e.hooks){if(!(n in t))throw new Error(`hook '${n}' does not exist`);if(["options","block"].includes(n))continue;const s=n,r=e.hooks[s],i=t[s];ae.passThroughHooks.has(n)?t[s]=e=>{if(this.defaults.async)return Promise.resolve(r.call(t,e)).then((e=>i.call(t,e)));const n=r.call(t,e);return i.call(t,n)}:t[s]=(...e)=>{let n=r.apply(t,e);return!1===n&&(n=i.apply(t,e)),n}}n.hooks=t}if(e.walkTokens){const t=this.defaults.walkTokens,s=e.walkTokens;n.walkTokens=function(e){let n=[];return n.push(s.call(this,e)),t&&(n=n.concat(t.call(this,e))),n}}this.defaults={...this.defaults,...n}})),this}setOptions(e){return this.defaults={...this.defaults,...e},this}lexer(e,t){return re.lex(e,t??this.defaults)}parser(e,t){return oe.parse(e,t??this.defaults)}parseMarkdown(e){return(t,n)=>{const s={...n},r={...this.defaults,...s},i=this.onError(!!r.silent,!!r.async);if(!0===this.defaults.async&&!1===s.async)return i(new Error("marked(): The async option was set to true by an extension. Remove async: false from the parse options object to return a Promise."));if(null==t)return i(new Error("marked(): input parameter is undefined or null"));if("string"!=typeof t)return i(new Error("marked(): input parameter is of type "+Object.prototype.toString.call(t)+", string expected"));r.hooks&&(r.hooks.options=r,r.hooks.block=e);const l=r.hooks?r.hooks.provideLexer():e?re.lex:re.lexInline,o=r.hooks?r.hooks.provideParser():e?oe.parse:oe.parseInline;if(r.async)return Promise.resolve(r.hooks?r.hooks.preprocess(t):t).then((e=>l(e,r))).then((e=>r.hooks?r.hooks.processAllTokens(e):e)).then((e=>r.walkTokens?Promise.all(this.walkTokens(e,r.walkTokens)).then((()=>e)):e)).then((e=>o(e,r))).then((e=>r.hooks?r.hooks.postprocess(e):e)).catch(i);try{r.hooks&&(t=r.hooks.preprocess(t));let e=l(t,r);r.hooks&&(e=r.hooks.processAllTokens(e)),r.walkTokens&&this.walkTokens(e,r.walkTokens);let n=o(e,r);return r.hooks&&(n=r.hooks.postprocess(n)),n}catch(e){return i(e)}}}onError(e,t){return n=>{if(n.message+="\nPlease report this to https://github.com/markedjs/marked.",e){const e="

    An error occurred:

    "+W(n.message+"",!0)+"
    ";return t?Promise.resolve(e):e}if(t)return Promise.reject(n);throw n}}}const he=new ce;function pe(e,t){return he.parse(e,t)}pe.options=pe.setOptions=function(e){return he.setOptions(e),pe.defaults=he.defaults,n(pe.defaults),pe},pe.getDefaults=t,pe.defaults=e.defaults,pe.use=function(...e){return he.use(...e),pe.defaults=he.defaults,n(pe.defaults),pe},pe.walkTokens=function(e,t){return he.walkTokens(e,t)},pe.parseInline=he.parseInline,pe.Parser=oe,pe.parser=oe.parse,pe.Renderer=ie,pe.TextRenderer=le,pe.Lexer=re,pe.lexer=re.lex,pe.Tokenizer=se,pe.Hooks=ae,pe.parse=pe;const ue=pe.options,ge=pe.setOptions,ke=pe.use,de=pe.walkTokens,fe=pe.parseInline,xe=pe,be=oe.parse,we=re.lex;e.Hooks=ae,e.Lexer=re,e.Marked=ce,e.Parser=oe,e.Renderer=ie,e.TextRenderer=le,e.Tokenizer=se,e.getDefaults=t,e.lexer=we,e.marked=pe,e.options=ue,e.parse=xe,e.parseInline=fe,e.parser=be,e.setOptions=ge,e.use=ke,e.walkTokens=de}));