commit 74b44d6644277fc5b0c68f89c376bdd22c3fcd3a Spenser Truex <struex0@gmail.com> 2018-08-01 14:13:52 -0700 Workaround added from broken cl+ssl.
README.md | 3 +++ librivox.asd | 9 ++++++++- packages.lisp | 53 +++++++++++++++++++++++++++++++++++-------------- rss/rss.lisp | 20 +++++++++---------- src/bash.lisp | 45 ++---------------------------------------- src/ffmpeg.lisp | 45 ++++++++++++++++++++++++++++++++++++++++++ src/librivox.lisp | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++- src/workaround.lisp | 3 +++ 8 files changed, 164 insertions(+), 71 deletions(-)
diff --git a/README.md b/README.md index c2ed1a9..2fc9c78 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # Requirements: +Foreign command line applications: `mp3info` `ffmpeg` `bash` +In your lisp: `Quicklisp` +`The LATEST CL+SSL, from the source repo. Quicklisp might not work.` \ No newline at end of file diff --git a/librivox.asd b/librivox.asd index 66a55e4..ac2095d 100644 --- a/librivox.asd +++ b/librivox.asd @@ -1,9 +1,16 @@ +(setf cl::*features* (cons :cl+ssl-broken cl::*features*)) +;;(setf *features* (cons :multithreading cl::*features*)) (asdf:defsystem :librivox :serial t - :depends-on (:drakma :uiop :cl-feedparser :bordeaux-threads) + :depends-on (#-cl+ssl-broken :drakma + :cl-feedparser + :uiop + :bordeaux-threads) :description "Librivox auto downloader/youtube uploader." :components ((:file "packages") (:file "utils") (:file "src/bash") + (:file "src/workaround") + (:file "src/ffmpeg") (:file "src/librivox"))) diff --git a/packages.lisp b/packages.lisp index 88d33dc..9a57e72 100644 --- a/packages.lisp +++ b/packages.lisp @@ -1,26 +1,49 @@ (defpackage :utils (:use :cl :cl-user) (:export :remove-after :nth-digit)) +(defpackage :bash + (:use :cl :uiop) + (:export :run-line + :*bash-output*)) +(defpackage :workaround + (:use :cl :bash) + (:import-from :bash + :run-line) + (:export :http-request)) (defpackage :rss - (:use :cl :drakma :utils :cl-feedparser) - (:import-from :drakma :http-request) + (:use :cl + #-cl+ssl-broken :drakma + #+cl+ssl-broken :workaround + :utils + :feedparser) + #-cl+ssl-broken (:import-from cl+ssl-broken :drakma :http-request) + #+cl+ssl-broken (:import-from :workaround :http-request) (:import-from :utils :remove-after) - ;;(:import-from :feedparser) - ;;(:export ) - ) + (:export :update! + :update + :parse-feed)) (defpackage :youtube - (:use :cl :drakma :utils)) - -(defpackage :bash - (:use :cl :uiop :utils) - (:import-from :utils :nth-digit) - (:export :convert - :processor-cores)) + (:use :cl + #+cl+ssl-broken :workaround + #-cl+ssl-broken :drakma + :utils) + (:import-from #+cl+ssl-broken :workaround + #-cl+ssl-broken :drakma + :http-request)) (defpackage :librivox - (:use :cl :drakma :utils :uiop :youtube :rss :bordeaux-threads) - (:import-from :uiop :run-program) - (:import-from :bash :convert :processor-cores) + (:use :cl + ;; Workaround for cl+ssl not working + #+cl+ssl-broken :workaround + #-cl+ssl-broken :drakma + :utils :uiop :youtube :rss :bordeaux-threads) + #+cl+ssl-broken (:import-from :workaround :http-request) + #-cl+ssl-broken (:import-from :drakma :http-request) + (:import-from :rss :parse-feed + :update + :update!) + (:import-from :utils :nth-digit) + (:import-from :bash :run-line :*bash-output*) (:import-from :bordeaux-threads :make-thread :make-lock diff --git a/rss/rss.lisp b/rss/rss.lisp index f6f0ad2..91fab74 100644 --- a/rss/rss.lisp +++ b/rss/rss.lisp @@ -1,18 +1,16 @@ (in-package :rss) (defvar *persistence* "persistent.lisp") -#|(let ((x (parse-feed "https://librivox.org/rss/latest_releases"))) - (loop for hk being the hash-key in x - for hv being the hash-value in x - collect (cons hk hv))) +(defun previous-update () + "Get the most recent update." + (with-open-file (s *persistence* + :direction :input + :if-does-not-exist nil) + (unless (null s) (read s nil nil)))) +(defun update! (url) + (previous-update)) (defun parse-feed (url) (feedparser:parse-feed (http-request url))) (defun update (url) (remove-after (previous-update) (readable-feed (parse-feed url)) - :test #'equal))|# -(defun previous-update () - "Get the most recent update." - (with-open-file (s *persistence* - :direction :input - :if-does-not-exist :create) - (read s nil nil))) + :test #'equal)) diff --git a/src/bash.lisp b/src/bash.lisp index 9a5ff5c..e79549e 100644 --- a/src/bash.lisp +++ b/src/bash.lisp @@ -1,45 +1,4 @@ (in-package :bash) (defvar *bash-output* *debug-io*) -(defvar *downloads-dir* "/home/jose/common-lisp/librivox/downloads/") -(defun run-line (code-string &optional (output *bash-output*)) - (run-program (format nil code-string *downloads-dir*) :output output)) -(defun run-line-integer-output (code-string) - (parse-integer (run-program (format nil code-string *downloads-dir*) - :output 'string) - :junk-allowed t)) -(defun hours (seconds) - (/ (- seconds (* 60 (minutes seconds)) (seconds seconds)) - 60 60)) -(defun minutes (seconds) - (nth-digit 1 seconds 60)) -(defun seconds (seconds) - (nth-digit 0 seconds 60)) -(defun processor-cores () - #+linux (run-line-integer-output "grep -c ^processor /proc/cpuinfo~*") - #-linux 1) -(defun convert (image-filename output-filename zip-filename) - "Convert the zip to video." - ;; ~A for the download directory, ~:*~A after the first. - ;; KLUDGE: if there is more than one zip - ;; KLUDGE: rm returns error code 1 for some reason, causing continuations. - ;; I would rather log those and automatically continue. - (ignore-errors (let ((output-filename (merge-pathnames output-filename *downloads-dir*)) - (image-filename (merge-pathnames image-filename *downloads-dir*)) - (zip-filename (merge-pathnames zip-filename *downloads-dir*))) - (run-line (format nil "unzip ~A -d ~~A" zip-filename)) - (run-line "for f in ~A*.mp3; do echo \"file '$f'\" >> ~:*~Aconcat.txt; done") - (run-line "ffmpeg -f concat -safe 0 -i ~Aconcat.txt -c copy ~:*~Aoutput1.mp3") - (let ((seconds (run-line-integer-output "mp3info -p \"%S\" ~Aoutput1.mp3"))) - (run-line (format nil "ffmpeg -loop 1 -i ~A -t ~D:~D:~D -c mjpeg ~~Atemp.mp4" ; - image-filename - (hours seconds) - (minutes seconds) - (seconds seconds))) - (run-line (format nil "ffmpeg -i ~~Atemp.mp4 -i ~~:*~~Aoutput1.mp3 -c copy ~A" - output-filename)) - (run-line "rm ~Atemp.mp4") - (run-line "rm ~A*.mp3") - (run-line (format nil "rm ~A ~A~~*" zip-filename image-filename)) - (run-line "rm ~Aconcat.txt"))))) -;;; (convert "abe_mawruss_1807.jpg" "final-test-probably1.mp4" "abeandmawruss_1807_librivox.zip") - +(defun run-line (code-string &key (output t)) + (run-program code-string :output output)) diff --git a/src/ffmpeg.lisp b/src/ffmpeg.lisp new file mode 100644 index 0000000..9316cc6 --- /dev/null +++ b/src/ffmpeg.lisp @@ -0,0 +1,45 @@ +(in-package :librivox) +(defvar *downloads-dir* "/home/jose/common-lisp/librivox/downloads/") +(defun run-line* (code-string &optional (output *bash-output*)) + (run-line (format nil code-string *downloads-dir*) :output output)) +(defun run-line*-integer-output (code-string) + (parse-integer (run-program (format nil code-string *downloads-dir*) + :output 'string) + :junk-allowed t)) +(defun hours (seconds) + (/ (- seconds (* 60 (minutes seconds)) (seconds seconds)) + 60 60)) +(defun minutes (seconds) + (nth-digit 1 seconds 60)) +(defun seconds (seconds) + (nth-digit 0 seconds 60)) +(defun processor-cores () + #+linux (run-line*-integer-output "grep -c ^processor /proc/cpuinfo~*") + #-linux 1) +(defun convert (&key image output zip) + "Convert the zip to video." + ;; ~A for the download directory, ~:*~A after the first. + ;; KLUDGE: if there is more than one zip + ;; KLUDGE: rm returns error code 1 for some reason, causing continuations. + ;; I would rather log those and automatically continue. + (ignore-errors (let ((output (merge-pathnames output *downloads-dir*)) + (image (merge-pathnames image *downloads-dir*)) + (zip (merge-pathnames zip *downloads-dir*))) + (run-line* (format nil "unzip ~A -d ~~A" zip)) + (run-line* "for f in ~A*.mp3; do echo \"file '$f'\" >> ~:*~Aconcat.txt; done") + (run-line* "ffmpeg -f concat -safe 0 -i ~Aconcat.txt -c copy ~:*~Aoutput1.mp3") + (let ((seconds (run-line*-integer-output "mp3info -p \"%S\" ~Aoutput1.mp3"))) + (run-line* (format nil "ffmpeg -loop 1 -i ~A -t ~D:~D:~D -c mjpeg ~~Atemp.mp4" ; + image + (hours seconds) + (minutes seconds) + (seconds seconds))) + (run-line* (format nil "ffmpeg -i ~~Atemp.mp4 -i ~~:*~~Aoutput1.mp3 -c copy ~A" + output)) + (run-line* "rm ~Atemp.mp4") + (run-line* "rm ~A*.mp3") + (run-line* "rm ~Aconcat.txt"))))) +;;; Example: +#|(bash:convert :image "abe_mawruss_1807.jpg" + :output "final-test-probably1.mp4" + :zip "abeandmawruss_1807_librivox.zip")|# diff --git a/src/librivox.lisp b/src/librivox.lisp index 2ce7fb9..41c24dc 100644 --- a/src/librivox.lisp +++ b/src/librivox.lisp @@ -1,3 +1,58 @@ (in-package :librivox) -(defparameter *max-threads* (processor-cores)) +;; As far as I can tell this is a common way of determining max threads. +;; FFMPEG does it automatically. +(defvar *downloads-dir* "/home/jose/common-lisp/librivox/downloads/") +(defparameter *max-threads* #+multithreading (processor-cores) + #- multithreading 1) (defvar *url* "https://librivox.org/rss/latest_releases") +(defun run-line* (code-string &optional (output *bash-output*)) + (run-line (format nil code-string *downloads-dir*) :output output)) +(defun run-line*-integer-output (code-string) + (parse-integer (run-program (format nil code-string *downloads-dir*) + :output 'string) + :junk-allowed t)) +(defun hours (seconds) + (/ (- seconds (* 60 (minutes seconds)) (seconds seconds)) + 60 60)) +(defun minutes (seconds) + (nth-digit 1 seconds 60)) +(defun seconds (seconds) + (nth-digit 0 seconds 60)) +(defun processor-cores () + #+(and linux multithreading) (run-line*-integer-output "grep -c ^processor /proc/cpuinfo~*") + #-linux 1 + #-multithreading 1) +(defun convert (&key image output zip) + "Convert the zip to video." + ;; ~A for the download directory, ~:*~A after the first. + ;; KLUDGE: Better to do a PART 1, PART 2, ETC system, since books are often >12 hours. + ;; KLUDGE: if there is more than one zip + ;; KLUDGE: rm returns error code 1 for some reason, causing continuations. + ;; I would rather log those and automatically continue. + (ignore-errors (let ((output (merge-pathnames output *downloads-dir*)) + (image (merge-pathnames image *downloads-dir*)) + (zip (merge-pathnames zip *downloads-dir*))) + (run-line* (format nil "unzip ~A -d ~~A" zip)) + (run-line* "for f in ~A*.mp3; do echo \"file '$f'\" >> ~:*~Aconcat.txt; done") + (run-line* "ffmpeg -f concat -safe 0 -i ~Aconcat.txt -c copy ~:*~Aoutput1.mp3") + (let ((seconds (run-line*-integer-output "mp3info -p \"%S\" ~Aoutput1.mp3"))) + (run-line* (format nil "ffmpeg -loop 1 -i ~A -t ~D:~D:~D -c mjpeg ~~Atemp.mp4" ; + image + (hours seconds) + (minutes seconds) + (seconds seconds))) + (run-line* (format nil "ffmpeg -i ~~Atemp.mp4 -i ~~:*~~Aoutput1.mp3 -c copy ~A" + output)) + (run-line* "rm ~Atemp.mp4") + (run-line* "rm ~A*.mp3") + (run-line* "rm ~Aconcat.txt"))))) +;;; Example: +#|(bash:convert :image "abe_mawruss_1807.jpg" + :output "final-test-probably1.mp4" + :zip "abeandmawruss_1807_librivox.zip")|# + +(if (= 1 *max-threads*) + ;; No multithreading: One book at a time. + (parse-feed (http-request *url*)) + ;; Multithreading: One thread per book method. + ()) diff --git a/src/workaround.lisp b/src/workaround.lisp new file mode 100644 index 0000000..db18726 --- /dev/null +++ b/src/workaround.lisp @@ -0,0 +1,3 @@ +(in-package :workaround) +(defun http-request (url) + (run-line (format nil "curl '~A'" url) :output 'string))