# HG changeset patch # User Arne Babenhauserheide # Date 1640653924 -3600 # Tue Dec 28 02:12:04 2021 +0100 # Node ID 9121e129547ac5dc262a52527e5f4a9f7fa1763b # Parent 20555a02fcbd68cce37be45552a51df4868409fd merge changes from FMS diff --git a/site/gms.scm b/site/gms.scm --- a/site/gms.scm +++ b/site/gms.scm @@ -6,8 +6,9 @@ ;;; gms.scm --- Guile Media Site - simple fake streaming via m3u playlists ;; Copyright (C) 2021 ArneBab +;; Copyright (C) 2021 Morrow_Singh@sJr0A3bn8e-NHUYydXFSyDDio~43O7m5fDBZUQj4lQY -;; Author: ArneBab +;; Author: ArneBab and Morrow Singh ;; This program is free software: you can redistribute it and/or modify ;; it under the terms of the GNU Affero General Public License as @@ -27,6 +28,12 @@ ;;; Code: +;; gms creates a video site. It selects random videos from ../media, +;; prepares them for video-on-demand, and creates an index.html and an +;; archive.html page with the videos. Each video is followed by random +;; selections of previous videos as with playlists on youtube. +;; It keeps at most 10 files to prevent endless growth of the site. + ;; Requirements: ;; - run on a GNU/Linux machine ;; - Guile @@ -37,13 +44,19 @@ ;; Usage: ;; - adjust style in template.html, video.html, audio.html, and style.css. ;; - put video and audio files into ../media/ -;; - run ./gms.scm to grab a random file from ../media/ and create an index.html site with all the already converted audio ready for streaming. +;; - run ./gms.scm to grab a random file from ../media/ and create an index.html site with all the already converted videos ready for streaming (up to 10 videos stay available). ;; - upload the folder containing gms.scm ;; - automatic weekly update: install pyFreenet. Then install a crontab-line with freesitemgr using `crontab -e`: ;; MINUTE HOUR * * WEEKDAY (sleep $((($RANDOM % 1800))) && cd path/to/site/ && nice ./gms.scm && ~/pyFreenet/freesitemgr --max-manifest-size=1500 update watch-36c3-incrementally) ;; replace minute and hour with the insert time and weekday with the day of week (1 is monday). The random sleep provides some obfuscation. ;; -;; ./gms.scm --rebuild just creates the site without converting and adding a new video. Use it for experimenting. +;; ./gms.scm --rebuild-only just creates the site without converting and adding a new video. Use it for experimenting. +;; ./gms.scm --recycle-removed puts removed files back into ../media/. Use it for infinite updates if you have more than 10 files. + +;;;; recreate all m3u-forwards +;;; remove the alternate m3u-lists +;; for i in *-stream.m3u; do grep -v .m3u "$i" > tmp && mv tmp "$i"; done; rm -f playlists; SAVEIFS=$IFS; IFS=$(echo -en "\n\b"); for i in $(ls --sort=time -r ../entries/); do touch "${i%%.*}"*; tac playlists | guile -c '(import (ice-9 rdelim))(set! *random-state* (random-state-from-platform))(let loop ((line (read-line))) (unless (eof-object? line) (when (< (random 5) 3) (display line)(newline)) (loop (read-line))))' >> "${i%%.*}"*-stream.m3u; echo "${i%%.*}"*-stream.m3u >> playlists; sleep 1; done; IFS=$SAVEIFS; rm -f playlists + ;; Approach: @@ -61,6 +74,9 @@ (ice-9 format) (srfi srfi-1)) +(define videos-on-first-page 2) +(define maximum-video-count 10) + (define-syntax-rule (read-first-line command) (let* ((port (open-input-pipe command)) (res (read-line port))) @@ -94,40 +110,55 @@ " ")) +(define (format-streamname filename) + (format #f "~a-stream.m3u" (entry-basename filename))) + +(define (entry-basename filename) + (basename filename (filename-extension filename))) + (define (convert-video filename) "Convert a video file to a freenet stream" (define name (basename filename)) - (define basename-without-extension (basename filename (filename-extension filename))) - (define streamname (format #f "~a-stream.m3u" basename-without-extension)) + (define basename-without-extension (entry-basename filename)) + (define streamname (format-streamname basename-without-extension)) (define start 0) - (define len 5) + (define len 9) ;; about 550k, so two files fit into the manifest. (define stop (+ start len)) (define (step) (set! start (+ start len)) - ;; compromise between linar increase for fast start and exponential increase for fewer breaks. 5 7 10 15 22 30 40 51 64 79 97 118 142 - (set! len (if (< len 11) (+ len (inexact->exact (truncate (/ len 2)))) (+ len 5 (inexact->exact (truncate (/ len 6)))))) + ;; exponential increase with larger initial segment in manifest to minimize breaks. + ;; 9 11 13 16 20 25 31 38 47 58 72 90 112 140 175 218 272 340 425 531 663 + (set! len (truncate (* len 5/4))) (set! stop (+ start len))) (define duration-seconds (inexact->exact - (string->number (read-first-line (format #f "ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 ~a" filename))))) + (string->number (read-first-line (format #f "ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 \"~a\"" filename))))) (define (ffmpeg index start stop) (when (< start duration-seconds) ;; skip early when the video is finished. (close-pipe (open-input-pipe - (format #f "ffmpeg -ss ~d -to ~d -accurate_seek -i ~a -y -g 360 -q:a 3 -q:v 3 -filter:v scale=640:-1 ~a-~3'0d.ogv" + (format #f "ffmpeg -threads 4 -ss ~d -to ~d -accurate_seek -i \"~a\" -y -g 360 -b:v 400k -b:a 56k -filter:v scale=720:-1 \"~a-~3'0d.ogv\"" start stop filename basename-without-extension index))))) ;; convert the video in segments (map (λ(x) (ffmpeg x start stop)(step)) (iota 999)) - (close-pipe (open-input-pipe (format #f "mplayer ~a -ss 5 -nosound -vf scale -zoom -xy 600 -vo jpeg:outdir=. -frames 1 && cp 00000001.jpg ~a.jpg" filename basename-without-extension))) + (close-pipe (open-input-pipe (format #f "mplayer \"~a\" -ss 5 -nosound -vf scale -zoom -xy 600 -vo jpeg:outdir=. -frames 1 && cp 00000001.jpg \"~a.jpg\"" filename basename-without-extension))) ;; move the file to the current directory if needed (when (not (equal? name filename)) - (close-pipe (open-input-pipe (format #f "mv ~a ~a" filename name)))) - ;; create stream playlist that continues with random other playlists after finishing. This might benefit from heuristics like sorting later streams by similarity to the original stream - (close-pipe (open-input-pipe (format #f "(ls ~a-*ogv; ls *-stream.m3u | shuf) > ~a" basename-without-extension streamname))) - (list (cons 'filename filename) - (cons 'basename name) - (cons 'first-chunk (format #f "ls ~a-*ogv | head -n 1" basename-without-extension)) - (cons 'streamname streamname) - (cons 'title (basename->title basename-without-extension)))) + (close-pipe (open-input-pipe (format #f "mv \"~a\" \"~a\"" filename name)))) + ;; create stream playlist that continues with random other playlists after finishing. This might benefit from heuristics like sorting later streams by similarity to the original stream. Skip the first two: the first is shown on the index page, the second can fail because its first segment was in the manifest, so it will only be inserted one insert later. + (close-pipe (open-input-pipe (format #f "(ls \"~a\"-*ogv; ls --sort=time *-stream.m3u | grep -v \"~a\" | tail +3 | guile -c '(import (ice-9 rdelim))(set! *random-state* (random-state-from-platform))(let loop ((line (read-line))) (unless (eof-object? line) (when (< (random 5) 3) (display line)(newline)) (loop (read-line))))') > \"~a\"" basename-without-extension streamname streamname))) + (entry-metadata filename)) + +(define (entry-metadata filename) + (define name (basename filename)) + (define basename-without-extension (entry-basename filename)) + (define streamname (format-streamname basename-without-extension)) + (let ((first-three (read-all-lines (format #f "ls \"~a-\"*ogv | head -n 2" basename-without-extension)))) + (list (cons 'filename filename) + (cons 'basename name) + (cons 'first-chunk (first first-three)) + (cons 'second-chunk (second first-three)) + (cons 'streamname streamname) + (cons 'title (basename->title basename-without-extension))))) ;; Guile 2 compat (define* (string-replace-substring s substr replacement #:optional (start 0) (end (string-length s))) @@ -147,33 +178,53 @@ (string-concatenate-reverse (cons (substring s start) pieces)))))))) - -(define (add-video next-video) - (define next-video-metadata (convert-video next-video)) + +(define (create-video-entry next-video-metadata) (define entry-template (read-file-as-string "video.html")) (define next-entry (string-replace-substring (string-replace-substring (string-replace-substring (string-replace-substring - (string-replace-substring - entry-template "{{{TITLE}}}" (assoc-ref next-video-metadata 'title)) - "{{{M3ULINK}}}" (assoc-ref next-video-metadata 'streamname)) - "{{{FIRSTCHUNK}}}" (assoc-ref next-video-metadata 'first-chunk)) + (string-replace-substring + (string-replace-substring + entry-template "{{{TITLE}}}" (assoc-ref next-video-metadata 'title)) + "{{{M3ULINK}}}" (assoc-ref next-video-metadata 'streamname)) + "{{{FIRSTCHUNK}}}" (assoc-ref next-video-metadata 'first-chunk)) + "{{{SECONDCHUNK}}}" (assoc-ref next-video-metadata 'second-chunk)) "{{{FILELINK}}}" (assoc-ref next-video-metadata 'basename)) "{{{FILENAME}}}" (assoc-ref next-video-metadata 'basename))) (let* ((port (open-output-file (string-append "../entries/" (assoc-ref next-video-metadata 'basename))))) (display next-entry port) - (close port)) + (close port))) + + +(define (add-video next-video) + (define next-video-metadata (convert-video next-video)) + (create-video-entry next-video-metadata) (display next-video) (newline) (sync)) -(define videos-on-first-page 2) +(define (recycle-video video-file) + (when (not (string-null? video-file)) + ;; move video back into media + (read-first-line (format #f "mv '~a' ../media/" video-file)))) + +(define (remove-video video-file) + (when (not (string-null? video-file)) + ;; delete stream and content files + (let ((cmd (format #f "rm '~a' \"~a-\"[0-9][0-9][0-9]\".ogv\"" (format-streamname video-file) (entry-basename video-file)))) + (display cmd) + (newline) + (read-first-line cmd)) + ;; remove stream from other streams + (read-first-line (format #f "for i in *-stream.m3u; do grep -v \"~a\" \"$i\" > ../tmpstream.m3u; mv ../tmpstream.m3u \"$i\"; done" (format-streamname video-file))) + (read-first-line (format #f "rm '../entries/~a'" video-file)))) (define (create-site) (define template (read-file-as-string "template.html")) - (define entry-filenames (map (λ (x) (string-append "../entries/" x)) (read-all-lines "ls --sort=time ../entries/"))) + (define entry-filenames (map (λ (x) (string-append "../entries/" x)) (read-all-lines (format #f "ls --sort=time ../entries/ | head -n ~a" maximum-video-count)))) (define entries (map read-file-as-string entry-filenames)) (define replaced-first (string-replace-substring template "{{{STREAMS}}}" (string-join (take entries (min videos-on-first-page (length entries))) "\n\n"))) (define replaced (string-replace-substring template "{{{STREAMS}}}" (string-join (drop entries (min videos-on-first-page (length entries))) "\n\n"))) @@ -187,9 +238,31 @@ (display replaced) (newline)) +(define (help args) + (format #t "~a [--help | --rebuild-only | --create-entry ] [--recycle-removed]\n" (car args))) + (define (main args) (define next-video (read-first-line "ls ../media/*.* | shuf | head -n 1")) - (when (not (member "--rebuild" args)) + (define help? (member "--help" args)) + (define rebuild-only? (member "--rebuild-only" args)) + (define create-entry? (member "--create-entry" args)) + (define recycle-removed-media? (member "--recycle-removed" args)) + (cond + (help? (help args)) + (create-entry? + (if (null? (cdr create-entry?)) + (help args) + (create-video-entry (entry-metadata (second create-entry?))))) + ((not rebuild-only?) (when (not (eof-object? next-video)) - (add-video next-video))) - (create-site)) + ;; recycle before removing the old videos + (when recycle-removed-media? + (let ((old-files (read-all-lines (format #f "ls --sort=time ../entries/ | tail -n +~a" (+ 1 maximum-video-count))))) + (map recycle-video old-files))) + ;; remove old videos before adding the new + (map remove-video (read-all-lines (format #f "ls --sort=time ../entries/ | tail -n +~a" (+ 1 maximum-video-count)))) + ;; create and add new video + (add-video next-video)) + (create-site)) + (else + (create-site)))) diff --git a/site/video.html b/site/video.html --- a/site/video.html +++ b/site/video.html @@ -1,5 +1,5 @@

{{{TITLE}}}

-