package supervisor import ( "encoding/json" "fmt" "io" "log" "os" "os/exec" "path/filepath" "runtime" "strconv" "strings" "sync" "time" "drmdecryption/apps/streamd/internal/db" "drmdecryption/mpd" ) // Media runs N_m3u8DL-RE + ffmpeg HLS packager per stream into DataDir/www//. type Media struct { Store *db.Store DataDir string NRE string FFmpeg string MP4Decrypt string Log *log.Logger mu sync.Mutex procs map[int64]*streamProc stopMon chan struct{} } type streamProc struct { id int64 name string nre *exec.Cmd ffmpeg *exec.Cmd started time.Time workDir string wwwDir string tsPath string hlsIndex string nreLog *os.File ffOutLog *os.File ffErrLog *os.File stopping bool mpdProxy *mpd.LocalServer nreDead bool ffDead bool // tsReadyBytes is how much muxed output to buffer before probing, from the // stream's app module. tsReadyBytes int64 } // Options for constructing the media supervisor. type Options struct { Store *db.Store DataDir string NRE string FFmpeg string MP4Decrypt string Log *log.Logger } func NewMedia(opt Options) *Media { lg := opt.Log if lg == nil { lg = log.Default() } m := &Media{ Store: opt.Store, DataDir: opt.DataDir, NRE: opt.NRE, FFmpeg: opt.FFmpeg, MP4Decrypt: opt.MP4Decrypt, Log: lg, procs: map[int64]*streamProc{}, stopMon: make(chan struct{}), } go m.monitorLoop() return m } func (m *Media) Close() { close(m.stopMon) m.mu.Lock() ids := make([]int64, 0, len(m.procs)) for id := range m.procs { ids = append(ids, id) } m.mu.Unlock() for _, id := range ids { _ = m.Stop(id) } } func (m *Media) Start(id int64) error { st, err := m.Store.GetStream(id) if err != nil { return err } if strings.TrimSpace(st.MPD) == "" || strings.TrimSpace(st.Key) == "" { _ = m.Store.SetRuntime(id, db.RuntimePatch{ Health: "down", PlayPath: "/hls/" + st.Name + "/index.m3u8", LastError: "missing mpd/key - capture required", }) return fmt.Errorf("missing mpd/key") } mpdURL := strings.TrimSpace(st.MPD) if !strings.HasPrefix(strings.ToLower(mpdURL), "http://") && !strings.HasPrefix(strings.ToLower(mpdURL), "https://") { _ = m.Store.SetRuntime(id, db.RuntimePatch{ Health: "down", PlayPath: "/hls/" + st.Name + "/index.m3u8", LastError: "mpd must be an http(s) URL", }) return fmt.Errorf("invalid mpd url") } if m.NRE == "" || m.FFmpeg == "" { _ = m.Store.SetRuntime(id, db.RuntimePatch{ Health: "down", PlayPath: "/hls/" + st.Name + "/index.m3u8", LastError: "NRE or ffmpeg binary not configured", }) return fmt.Errorf("NRE or ffmpeg not configured") } m.mu.Lock() if _, exists := m.procs[id]; exists { m.mu.Unlock() return m.Restart(id) } m.mu.Unlock() _ = m.Store.SetRuntime(id, db.RuntimePatch{ Health: "starting", PlayPath: "/hls/" + st.Name + "/index.m3u8", }) workDir := filepath.Join(m.DataDir, "work", st.Name) wwwDir := filepath.Join(m.DataDir, "www", st.Name) // Fresh dirs each start — leftover decrypted segments make NRE File.Move fail. _ = os.RemoveAll(wwwDir) _ = os.RemoveAll(workDir) for _, d := range []string{workDir, wwwDir, filepath.Join(m.DataDir, "logs")} { if err := os.MkdirAll(d, 0o755); err != nil { return err } } keysFile := filepath.Join(workDir, "keys.txt") keysBody := normalizeKeysFile(st.Key) if err := os.WriteFile(keysFile, []byte(keysBody), 0o644); err != nil { return err } saveName := st.Name tsPath := filepath.Join(workDir, saveName+".ts") _ = os.Remove(tsPath) headers := headerMap(st.HeadersJSON) manifestURL := st.MPD cfg := settingsFor(st) var mpdProxy *mpd.LocalServer // The rewriter is named by the stream row or its app module and looked up in // the mpd registry, so streamd needs no knowledge of any provider. if rw, needed := mpd.Lookup(cfg.Rewriter); needed && strings.TrimSpace(st.MPD) != "" && !cfg.IsHLS { srv, err := mpd.StartLocal(st.MPD, st.Key, headers, rw) if err != nil { _ = m.Store.SetRuntime(id, db.RuntimePatch{ Health: "down", PlayPath: "/hls/" + st.Name + "/index.m3u8", LastError: "mpd rewrite server: " + err.Error(), }) return err } mpdProxy = srv manifestURL = srv.URL m.logf("stream %s: rewritten MPD %s", st.Name, manifestURL) } // Ad-stitched manifest rewrite + live pipe mux. nreTmp := filepath.Join(workDir, "nre") _ = os.MkdirAll(nreTmp, 0o755) tsPath = filepath.Join(nreTmp, saveName+".ts") _ = os.Remove(tsPath) liveWait := strconv.Itoa(cfg.LiveWait) nreArgs := []string{ manifestURL, // Multi-KID SAMPLE-AES streams rainbow-decrypt with a single --key, so // always feed every KID:KEY via --key-text-file. "--key-text-file", keysFile, "--live-real-time-merge", "--live-pipe-mux", "--mp4-real-time-decryption", "--live-wait-time", liveWait, "--ffmpeg-binary-path", m.FFmpeg, "--save-name", saveName, "--save-dir", nreTmp, "--tmp-dir", nreTmp, "--no-ansi-color", "--log-level", "ERROR", "-ss", "0", } // A synthetic manifest publishes exactly one video + audio pair, and an HLS // master's variants rarely match a resolution/language filter, so in both // cases take the best rendition instead of the configured selectors. if mpdProxy != nil || cfg.IsHLS { nreArgs = append(nreArgs, "-sv", "for=best", "-sa", "for=best") } else { nreArgs = append(nreArgs, "-sv", cfg.VideoSelect, "-sa", cfg.AudioSelect) } if m.MP4Decrypt != "" { nreArgs = append(nreArgs, "--decryption-engine", "MP4DECRYPT", "--decryption-binary-path", m.MP4Decrypt, ) } // Headers only needed when hitting upstream DAI directly (non-rewritten). if mpdProxy == nil { for _, h := range headerFlags(st.HeadersJSON) { nreArgs = append(nreArgs, "-H", h) } } _ = keysFile // kept on disk for debugging nreLogPath := filepath.Join(m.DataDir, "logs", st.Name+"-nre.log") nreLog, err := os.OpenFile(nreLogPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) if err != nil { return err } nreCmd := exec.Command(m.NRE, nreArgs...) nreCmd.Stdout = nreLog nreCmd.Stderr = nreLog hideWindow(nreCmd) // NRE (v0.6) resolves mp4decrypt via PATH / sibling of its .exe, and often // ignores --decryption-binary-path for MP4DECRYPT. Match legacy: prepend // dirs and best-effort copy beside NRE. nreEnv := os.Environ() pathPrepend := []string{} if m.MP4Decrypt != "" { pathPrepend = append(pathPrepend, filepath.Dir(m.MP4Decrypt)) sibling := filepath.Join(filepath.Dir(m.NRE), filepath.Base(m.MP4Decrypt)) if _, err := os.Stat(sibling); err != nil { _ = copyFile(m.MP4Decrypt, sibling) } } if m.NRE != "" { pathPrepend = append(pathPrepend, filepath.Dir(m.NRE)) } if len(pathPrepend) > 0 { sep := string(os.PathListSeparator) nreEnv = prependPathEnv(nreEnv, strings.Join(pathPrepend, sep)) } nreCmd.Env = nreEnv if err := nreCmd.Start(); err != nil { _ = nreLog.Close() if mpdProxy != nil { mpdProxy.Close() } _ = m.Store.SetRuntime(id, db.RuntimePatch{Health: "down", LastError: err.Error(), PlayPath: "/hls/" + st.Name + "/index.m3u8"}) return err } m.logf("stream %s: NRE pid=%d", st.Name, nreCmd.Process.Pid) sp := &streamProc{ id: id, name: st.Name, nre: nreCmd, started: time.Now(), workDir: workDir, wwwDir: wwwDir, tsPath: tsPath, hlsIndex: filepath.Join(wwwDir, "index.m3u8"), nreLog: nreLog, mpdProxy: mpdProxy, tsReadyBytes: cfg.TSReadyBytes, } m.mu.Lock() m.procs[id] = sp m.mu.Unlock() go func() { _ = nreCmd.Wait() m.mu.Lock() if cur, ok := m.procs[id]; ok && cur.nre == nreCmd { cur.nreDead = true } m.mu.Unlock() }() go m.bootstrapHLS(sp) return nil } func (m *Media) bootstrapHLS(sp *streamProc) { deadline := time.Now().Add(3 * time.Minute) nreDir := filepath.Dir(sp.tsPath) minTS := sp.tsReadyBytes for time.Now().Before(deadline) { m.mu.Lock() cur := m.procs[sp.id] stopping := cur == nil || cur.stopping nreDead := sp.nreDead m.mu.Unlock() if stopping { return } if found := findGrowingTSMin(nreDir, sp.name, minTS); found != "" { sp.tsPath = found m.mu.Lock() if cur, ok := m.procs[sp.id]; ok { cur.tsPath = found } m.mu.Unlock() break } if nreDead || (sp.nre.ProcessState != nil && sp.nre.ProcessState.Exited()) { _ = m.Store.SetRuntime(sp.id, db.RuntimePatch{ Health: "down", PlayPath: "/hls/" + sp.name + "/index.m3u8", LastError: "NRE exited before TS was ready — see logs/" + sp.name + "-nre.log", }) m.cleanupProc(sp.id) return } time.Sleep(500 * time.Millisecond) } if st, err := os.Stat(sp.tsPath); err != nil || st.Size() < minTS { _ = m.Store.SetRuntime(sp.id, db.RuntimePatch{ Health: "down", PlayPath: "/hls/" + sp.name + "/index.m3u8", LastError: "timed out waiting for growing TS", }) _ = m.Stop(sp.id) return } if err := m.startFFmpegHLS(sp); err != nil { _ = m.Store.SetRuntime(sp.id, db.RuntimePatch{ Health: "down", LastError: err.Error(), PlayPath: "/hls/" + sp.name + "/index.m3u8", }) return } pid := int64(0) if sp.nre != nil && sp.nre.Process != nil { pid = int64(sp.nre.Process.Pid) } _ = m.Store.SetRuntime(sp.id, db.RuntimePatch{ Health: "starting", PID: pid, PlayPath: "/hls/" + sp.name + "/index.m3u8", }) } func (m *Media) startFFmpegHLS(sp *streamProc) error { ffOut := filepath.Join(m.DataDir, "logs", sp.name+"-ffmpeg.out.log") ffErr := filepath.Join(m.DataDir, "logs", sp.name+"-ffmpeg.err.log") outF, err := os.OpenFile(ffOut, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) if err != nil { return err } errF, err := os.OpenFile(ffErr, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) if err != nil { _ = outF.Close() return err } ffArgs := []string{ "-hide_banner", "-loglevel", "warning", "-y", "-follow", "1", // Large probe: early pipe-mux bytes often lack clean audio config. "-analyzeduration", "20M", "-probesize", "10M", "-fflags", "+genpts", "-i", sp.tsPath, "-map", "0", "-c", "copy", "-f", "hls", "-hls_time", "4", "-hls_list_size", "15", "-hls_flags", "delete_segments+append_list+omit_endlist+independent_segments", "-hls_segment_type", "mpegts", "-hls_segment_filename", filepath.Join(sp.wwwDir, "seg_%05d.ts"), sp.hlsIndex, } ffCmd := exec.Command(m.FFmpeg, ffArgs...) ffCmd.Stdout = outF ffCmd.Stderr = errF hideWindow(ffCmd) if err := ffCmd.Start(); err != nil { _ = outF.Close() _ = errF.Close() return err } m.logf("stream %s: ffmpeg HLS pid=%d → %s", sp.name, ffCmd.Process.Pid, sp.hlsIndex) m.mu.Lock() if cur, ok := m.procs[sp.id]; ok && !cur.stopping { if cur.ffOutLog != nil { _ = cur.ffOutLog.Close() } if cur.ffErrLog != nil { _ = cur.ffErrLog.Close() } cur.ffmpeg = ffCmd cur.ffOutLog = outF cur.ffErrLog = errF cur.ffDead = false } else { m.mu.Unlock() _ = killProcess(ffCmd) _ = outF.Close() _ = errF.Close() return fmt.Errorf("stream stopped before ffmpeg attach") } m.mu.Unlock() go func() { _ = ffCmd.Wait() m.mu.Lock() if cur, ok := m.procs[sp.id]; ok && cur.ffmpeg == ffCmd { cur.ffDead = true } m.mu.Unlock() }() return nil } func (m *Media) Stop(id int64) error { m.mu.Lock() sp, ok := m.procs[id] if ok { sp.stopping = true } m.mu.Unlock() if !ok { st, _ := m.Store.GetStream(id) name := fmt.Sprintf("%d", id) if st.Name != "" { name = st.Name } _ = m.Store.SetRuntime(id, db.RuntimePatch{Health: "stopped", PlayPath: "/hls/" + name + "/index.m3u8"}) return nil } nrePID, ffPID := 0, 0 if sp.ffmpeg != nil && sp.ffmpeg.Process != nil { ffPID = sp.ffmpeg.Process.Pid _ = killProcess(sp.ffmpeg) } if sp.nre != nil && sp.nre.Process != nil { nrePID = sp.nre.Process.Pid _ = killProcess(sp.nre) } if sp.mpdProxy != nil { sp.mpdProxy.Close() sp.mpdProxy = nil } waitPIDsGone([]int{nrePID, ffPID}, 5*time.Second) m.cleanupProc(id) _ = m.Store.SetRuntime(id, db.RuntimePatch{ Health: "stopped", PlayPath: "/hls/" + sp.name + "/index.m3u8", }) m.logf("stream %s: stopped", sp.name) return nil } func (m *Media) Restart(id int64) error { m.logf("stream id=%d: hard restart (kill worker, fresh start)", id) _ = m.Stop(id) // Give Windows time to release file locks on work/www before Start wipes them. time.Sleep(1 * time.Second) return m.Start(id) } func (m *Media) cleanupProc(id int64) { m.mu.Lock() defer m.mu.Unlock() sp, ok := m.procs[id] if !ok { return } if sp.nreLog != nil { _ = sp.nreLog.Close() } if sp.ffOutLog != nil { _ = sp.ffOutLog.Close() } if sp.ffErrLog != nil { _ = sp.ffErrLog.Close() } delete(m.procs, id) } func (m *Media) monitorLoop() { t := time.NewTicker(5 * time.Second) defer t.Stop() for { select { case <-m.stopMon: return case <-t.C: m.mu.Lock() ids := make([]int64, 0, len(m.procs)) for id := range m.procs { ids = append(ids, id) } m.mu.Unlock() for _, id := range ids { m.refreshHealth(id) } } } } func (m *Media) refreshHealth(id int64) { m.mu.Lock() sp, ok := m.procs[id] m.mu.Unlock() if !ok || sp.stopping { return } m.mu.Lock() nreDead := sp.nreDead ffDead := sp.ffDead m.mu.Unlock() playPath := "/hls/" + sp.name + "/index.m3u8" uptime := time.Since(sp.started).Seconds() pid := int64(0) if sp.nre != nil && sp.nre.Process != nil { pid = int64(sp.nre.Process.Pid) } if nreDead { _ = m.Store.SetRuntime(id, db.RuntimePatch{ Health: "down", PID: 0, PlayPath: playPath, UptimeS: uptime, LastError: "NRE process exited — see logs/" + sp.name + "-nre.log", }) m.cleanupProc(id) return } fi, err := os.Stat(sp.hlsIndex) if err != nil { _ = m.Store.SetRuntime(id, db.RuntimePatch{ Health: "starting", PID: pid, PlayPath: playPath, UptimeS: uptime, LastError: "waiting for HLS playlist", }) if ffDead { // restart ffmpeg packager if TS still growing go m.restartFFmpeg(sp) } return } age := time.Since(fi.ModTime()).Seconds() health := "ok" lastErr := "" if age > 45 { health = "down" lastErr = fmt.Sprintf("playlist stale (%.0fs)", age) } _ = m.Store.SetRuntime(id, db.RuntimePatch{ Health: health, PID: pid, PlayPath: playPath, UptimeS: uptime, PlaylistAgeS: age, LastError: lastErr, }) if ffDead && health != "down" { go m.restartFFmpeg(sp) } } func (m *Media) restartFFmpeg(sp *streamProc) { m.mu.Lock() cur, ok := m.procs[sp.id] if !ok || cur.stopping || !cur.ffDead { m.mu.Unlock() return } // Claim the restart slot so monitorLoop does not spawn duplicates. cur.ffDead = false if cur.ffmpeg != nil { _ = killProcess(cur.ffmpeg) } tsPath := cur.tsPath m.mu.Unlock() minTS := sp.tsReadyBytes if st, err := os.Stat(tsPath); err != nil || st.Size() < minTS { m.mu.Lock() if cur, ok := m.procs[sp.id]; ok { cur.ffDead = true } m.mu.Unlock() return } m.logf("stream %s: restarting ffmpeg HLS packager", sp.name) if err := m.startFFmpegHLS(sp); err != nil { m.logf("stream %s: ffmpeg restart failed: %v", sp.name, err) m.mu.Lock() if cur, ok := m.procs[sp.id]; ok { cur.ffDead = true } m.mu.Unlock() } } func (m *Media) logf(format string, args ...any) { m.Log.Printf("supervisor: "+format, args...) } func headerMap(headersJSON string) map[string]string { headersJSON = strings.TrimSpace(headersJSON) if headersJSON == "" || headersJSON == "{}" { return nil } var m map[string]string if err := json.Unmarshal([]byte(headersJSON), &m); err != nil { return nil } out := map[string]string{} for k, v := range m { k = strings.TrimSpace(k) v = strings.TrimSpace(v) if k != "" && v != "" { out[k] = v } } return out } func headerFlags(headersJSON string) []string { m := headerMap(headersJSON) out := make([]string, 0, len(m)) for k, v := range m { out = append(out, k+": "+v) } return out } // normalizeKeysFile turns a DB key field (one or many KID:KEY lines, or // comma/semicolon separated) into an NRE --key-text-file body. func normalizeKeysFile(raw string) string { raw = strings.ReplaceAll(raw, ",", "\n") raw = strings.ReplaceAll(raw, ";", "\n") var lines []string seen := map[string]bool{} for _, line := range strings.Split(raw, "\n") { line = strings.TrimSpace(line) if line == "" || strings.Count(line, ":") != 1 { continue } if seen[line] { continue } seen[line] = true lines = append(lines, line) } if len(lines) == 0 { return strings.TrimSpace(raw) + "\n" } return strings.Join(lines, "\n") + "\n" } func killProcess(cmd *exec.Cmd) error { if cmd == nil || cmd.Process == nil { return nil } pid := cmd.Process.Pid if runtime.GOOS == "windows" { _ = exec.Command("taskkill", "/T", "/F", "/PID", strconv.Itoa(pid)).Run() return nil } return cmd.Process.Kill() } func prependPathEnv(env []string, prefix string) []string { if prefix == "" { return env } out := make([]string, 0, len(env)+1) found := false for _, e := range env { if len(e) >= 5 && strings.EqualFold(e[:5], "PATH=") { out = append(out, "PATH="+prefix+string(os.PathListSeparator)+e[5:]) found = true continue } out = append(out, e) } if !found { out = append(out, "PATH="+prefix) } return out } func copyFile(src, dst string) error { in, err := os.Open(src) if err != nil { return err } defer in.Close() out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755) if err != nil { return err } defer out.Close() if _, err := io.Copy(out, in); err != nil { return err } return out.Close() } // findGrowingTS locates NRE's live-pipe-mux output. Depending on version/flags // it may be saveName.ts or saveName..ts (e.g. test.en.ts). func waitPIDsGone(pids []int, timeout time.Duration) { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { alive := false for _, pid := range pids { if pid <= 0 { continue } if !processDead(pid) { alive = true break } } if !alive { return } time.Sleep(100 * time.Millisecond) } } func findGrowingTS(dir, saveName string) string { return findGrowingTSMin(dir, saveName, 256*1024) } func findGrowingTSMin(dir, saveName string, minSize int64) string { if minSize <= 0 { minSize = 256 * 1024 } candidates := []string{ filepath.Join(dir, saveName+".ts"), } if ents, err := os.ReadDir(dir); err == nil { prefix := saveName + "." for _, e := range ents { if e.IsDir() { continue } name := e.Name() if !strings.HasSuffix(strings.ToLower(name), ".ts") { continue } if name == saveName+".ts" || strings.HasPrefix(name, prefix) { candidates = append(candidates, filepath.Join(dir, name)) } } } var best string var bestSize int64 for _, p := range candidates { st, err := os.Stat(p) if err != nil || st.Size() < minSize { continue } if st.Size() > bestSize { bestSize = st.Size() best = p } } return best }