commit 61ddc40eff416fe620557bc828f4000c58c31dca Spenser Truex <spensertruexonline@gmail.com> 2019-01-07 15:52:42 -0800 Code refactoring, splitting to testing fork.
librivox.asd | 12 ++++---- packages.lisp | 50 ++++++++++++++++---------------- src/bash.lisp | 5 +--- src/csv.lisp | 48 ------------------------------- src/ffmpeg.lisp | 85 +++++++++++++++++++++++++++++-------------------------- src/librivox.lisp | 16 +++++++---- 6 files changed, 86 insertions(+), 130 deletions(-)
diff --git a/librivox.asd b/librivox.asd index 3d5d0f8..832c12d 100644 --- a/librivox.asd +++ b/librivox.asd @@ -1,18 +1,16 @@ -(setf cl::*features* (cons :cl+ssl-broken cl::*features*)) +(push :cl+ssl-broken cl::*features*) ;;(setf *features* (cons :multithreading cl::*features*)) (asdf:defsystem :librivox :serial t :depends-on (#-cl+ssl-broken :drakma - :cl-ppcre - :uiop - :bordeaux-threads - :utils) + :cl-ppcre :uiop :bordeaux-threads :utils :csv) :description "Librivox auto downloader/youtube uploader." + :author "Spenser Truex spensertruex.com" :components ((:file "packages") - (:file "src/csv") (:file "src/bash") (:file "src/workaround") (:file "src/ffmpeg") - (:fiel "src/uploader") + (:file "src/uploader") + (:file "src/csv-specific") (:file "src/librivox"))) diff --git a/packages.lisp b/packages.lisp index 7ef1aa4..fb12a86 100644 --- a/packages.lisp +++ b/packages.lisp @@ -1,14 +1,5 @@ -(defpackage :csv - (:use :cl :utils)) -(defpackage :bash - (:use :cl :uiop) - (:export :run-line - :run-line-integer-output - :*bash-output*)) (defpackage :workaround - (:use :cl :bash) - (:import-from :bash - :run-line) + (:use :cl) (:export :http-request)) (defpackage :librivox (:use :cl @@ -18,20 +9,27 @@ :utils :uiop :bordeaux-threads) #+cl+ssl-broken (:import-from :workaround :http-request) #-cl+ssl-broken (:import-from :drakma :http-request) - (:import-from :utils :mvbind :dbind :dolines - :with-gensyms :once-only :aif :awhen :select) - (:import-from :bash :run-line :*bash-output* :run-line-integer-output) + (:import-from + :utils + :mvbind :dbind :dolines + :with-gensyms + :once-only + :aif + :awhen + :select + :run-line) (:import-from :uiop :subdirectories :directory-files) - (:import-from :bordeaux-threads - :make-thread - :make-lock - :join-thread - :thread-yield - :make-condition-variable - :condition-wait - :condition-notify - :all-threads - :thread-alive-p - :destroy-thread - :interrupt-thread - :join-thread)) + (:import-from + :bordeaux-threads + :make-thread + :make-lock + :join-thread + :thread-yield + :make-condition-variable + :condition-wait + :condition-notify + :all-threads + :thread-alive-p + :destroy-thread + :interrupt-thread + :join-thread)) diff --git a/src/bash.lisp b/src/bash.lisp index aad307b..8b13789 100644 --- a/src/bash.lisp +++ b/src/bash.lisp @@ -1,4 +1 @@ -(in-package :bash) -(defvar *bash-output* *debug-io*) -(defun run-line (code &key (output t) (ignore-error-status t)) - (run-program code :output output :ignore-error-status ignore-error-status)) + diff --git a/src/csv.lisp b/src/csv.lisp deleted file mode 100644 index 08d769a..0000000 --- a/src/csv.lisp +++ /dev/null @@ -1,48 +0,0 @@ -(in-package :csv) -(defun make-escape? (prev curr) - (char= #\" prev curr) - ;(char= #\\ prev) - ) -(defun end-token? (prev curr) - (declare (ignore curr)) - (char= #\" prev)) -(defun open? (prev curr) - (declare (ignore curr)) - (char= #\" prev)) -(let ((in nil) - (escape nil)) - (setf (fdefinition 'switches) - (lambda (prev curr str) - (if (null curr) - (progn (setf in nil escape nil) str) - (if in - (if escape - (progn (toggle escape) (push-on prev str)) - (if (make-escape? prev curr) - (progn (toggle escape) str) - (if (end-token? prev curr) - (progn (toggle in) (push-on #\Nul str)) - (push-on prev str)))) - (if (open? prev curr) - (progn (toggle in) str) - str)))))) -(defun splice (str &optional acc!) - (if (string= "" str) - (nreverse acc!) - (let ((res (search (string #\Nul) str))) - (splice (subseq str (1+ res)) (push (subseq str 0 res) acc!))))) -(make-reader my-csv (stream #\# #\!) - (macrolet ((once (&body body) ``',,@body)) - (do ((prev #1= (read-char stream nil nil) curr) - (curr #1# #1#) - (str (make-array 0 :adjustable t :element-type 'character) - (switches prev curr str))) - ((and (char= curr #\#) (char= prev #\!)) (once (splice (switches nil nil str))))))) -(defun read-to-string (stream) - (with-output-to-string (s) - (dolines (line stream) - (princ line s) - (format s "~%")))) -(defun slurp-csv (path) - (with-open-file (s path) - (concatenate 'string "#!" (read-to-string s) "!#"))) diff --git a/src/ffmpeg.lisp b/src/ffmpeg.lisp index 4997920..132245f 100644 --- a/src/ffmpeg.lisp +++ b/src/ffmpeg.lisp @@ -4,62 +4,67 @@ (defvar *image-types* (list "jpg")) (defvar *archive-types* (list "m3u")) (defvar *video-types* (list "mp4")) +(defvar *audio-types* (list "mp3")) +(defvar *text-types* (list "txt")) +(defvar *types* (list *image-types* + *archive-types* + *video-types* + *audio-types* + *text-types*)) +(defvar *comletion-name* "completion.tmp") (defun change-dir (dir code) (format nil "cd ~A && ~A" dir code)) -(defun run-line* (code &optional (output *bash-output*)) +(defun run-line* (code &optional (output *debug-io*)) (run-line (change-dir *downloads-dir* code) :output output)) (defun run-line*-integer-output (code) (parse-integer (run-program (change-dir *downloads-dir* code) :output :string) :junk-allowed t)) -(defun m3u-down (path) +(defun book-down (path) (with-open-file (s path) (dolines (line s) (run-line* (format nil "wget -nc ~A" line)) (print line)))) -(defmacro with-dir (dir &body body) - (with-gensyms (old) - `(let ((,old ,*downloads-dir*)) - (setf *downloads-dir* ,dir) - ,@body - (setf *downloads-dir* ,old)))) (defmacro unwind-prog1 (cleanup &body protected) `(unwind-protect (progn ,@protected) ,cleanup)) -(defun convert (&key image input (dir *downloads-dir*)) +(defun delete-types (&optional more) + (dolist (y (append *types* more)) + (run-line* (concat "rm *." y)))) +(defun safe-exit (old-dir) + (setf *downloads-dir* old-dir) + (delete-types)) +(defun convert (&key dir image input) "Convert the zip to video." (let ((old-dir *downloads-dir*)) (setf *downloads-dir* dir) (unwind-prog1 - (progn (run-line* "rm *.mp4") - (run-line* "rm *.mp3") - (run-line* "rm concat.txt") - (setf *downloads-dir* old-dir)) - (m3u-down input) - (run-line* "for f in *.mp3; do echo \"file '$f'\" >> concat.txt; done") - (run-line* "ffmpeg -f concat -safe 0 -i concat.txt -c copy output1.mp3") - (mvbind (times rem) (ceiling (/ (run-line*-integer-output "mp3info -p \"%S\" output1.mp3") *max-seconds*)) - (dotimes (n times) - (let ((duration (if (= times (1+ n)) - (* (1+ rem) *max-seconds*) - *max-seconds*))) - (run-line* (format nil "ffmpeg -loop 1 -i ~A -t ~D -c mjpeg temp-~D.mp4" - image duration n)) - (run-line* (format nil "ffmpeg -ss ~D -t ~D -i output1.mp3 -c copy output-~D.mp3" - (* n *max-seconds*) duration n)) - (run-line* (format nil "ffmpeg -i temp-~D.mp4 -i output-~:*~D.mp3 -c copy output-~:*~D.mp4" - n)) - (run-line* (format nil "rm temp-~D.mp4" n)))))))) -(defun pathname-type? (path type) - (string-equal (pathname-type path) type)) -(defun image-type? (path) - (some #'(lambda (x) (string-equal (pathname-type path) x)) *image-types*)) -(defun archive-type? (path) - (some #'(lambda (x) (string-equal (pathname-type path) x)) *archive-types*)) -(defun video-type? (path) - (some #'(lambda (x) (string-equal (pathname-type path) x)) *video-types*)) + (safe-exit old-dir) + (book-down input) + (run-line* "for f in *.mp3; do echo \"file '$f'\" >> concat.txt; done") + (run-line* "ffmpeg -f concat -safe 0 -i concat.txt -c copy output1.mp3") + (mvbind (times rem) (ceiling (/ (run-line*-integer-output "mp3info -p \"%S\" output1.mp3") *max-seconds*)) + (dotimes (n times) + (let ((duration (if (= times (1+ n)) + (* (1+ rem) *max-seconds*) + *max-seconds*))) + (run-line* (format nil "ffmpeg -loop 1 -i ~A -t ~D -c mjpeg temp-~D.mp4" + image duration n)) + (run-line* (format nil "ffmpeg -ss ~D -t ~D -i output1.mp3 -c copy output-~D.mp3" + (* n *max-seconds*) duration n)) + (run-line* (format nil "ffmpeg -i temp-~D.mp4 -i output-~:*~D.mp3 -c copy output-~:*~D.mp4" + n)) + (run-line* (format nil "rm temp-~D.mp4" n)))))))) +(defmacro deftypep (name general &key (test 'string-equal)) + "Specific file type check." + (with-gensyms (x) + `(defun ,name (path) + (some #'(lambda (,x) (funcall ,test (pathname-type path) ,x)) + ,general)))) +(deftypep image-typep *image-types*) +(deftypep archive-typep *archive-types*) +(deftypep video-typep *video-types*) (defun convert-all () (dolist (dir (subdirectories *downloads-dir*)) (let ((contents (directory-files dir))) - (when (not (some #'video-type? - contents)) - (convert :image (select #'image-type? contents) - :input (select #'archive-type? contents) + (when (not (some #'video-typep contents)) + (convert :image (select #'image-typep contents) + :input (select #'archive-typep contents) :dir dir))))) diff --git a/src/librivox.lisp b/src/librivox.lisp index db180a7..bfaa958 100644 --- a/src/librivox.lisp +++ b/src/librivox.lisp @@ -1,7 +1,13 @@ (in-package :librivox) - -(defun start (csv-input) +(defun start () "Just start everything: download, convert, upload." - (download) - (convert) - (upload)) + (csv-expand (csv-update) :forward t) + (convert-all)) + +(defun csv-update () + "Get a new CSV to keep up with recent audiobook publishing.") +(defun csv-expand (&key forward) + "Convert a CSV of librivox audiobooks into a directory tree filled with files +to be used in the downloading and conversion portions. `FORWARD' determines if +the directory tree is being made fresh -> nil or just updated -> t. " + )