(import httprequest :as r) (import spork/json :as json) (import ./media :as media) (def apptoken "REDACTED") (def urls @["https://hachyderm.io/api/v1/bookmarks"]) (def save-path "./bookmarks/") (def done @[]) (defn array/shift [arr] "Removes the first element from the array." (if (empty? arr) (break nil)) (def head (in arr 0)) (array/remove arr 0) head) (defn get-link [response rel] "Gets the link with the specified relation from the response. e.g. (get-link response \"next\")" # Get the links from the headers (def link-header ((response :headers) "Link")) (if (= link-header nil) (break nil)) (def links (string/split "," link-header)) # Find the link with the specified relation (def link (find |(not= (string/find rel $) nil) links)) # Extract the url from the link string (as an array of 1 element) (def url (peg/match ~(sequence (any " ") "<" (capture (some (if-not ">" 1))) ">") link)) (if url (in url 0) nil)) (defn download-media [item] "Downloads the media from the item." (def files @[]) (def media (item "media_attachments")) (each m media (def url (m "url")) (def folder (string save-path "attachments/")) (array/push files {:path (string "./attachments/" (media/get-image url folder)) :description (m "description")})) (pp files) files) (defn write-markdown [item] (def files (download-media item)) (def id (item "id")) (def fout (file/open (string save-path id ".md") :w)) (def inline (string "mastodon-url:: " (item "url") " \n" "mastodon-user:: " "[" ((item "account") "acct") "]" "(" ((item "account") "url") ")" " \n" "mastodon-published:: " (item "created_at") " \n" "mastodon-tags:: " (string/join (map (fn [tag] (string "[" (tag "name") "]" "(" (tag "url") ")")) (item "tags")) " ") " \n")) (def media (string/join (map (fn [arg] (string "![](" (arg :path) ")\n " (arg :description))) files) " ")) (file/write fout (string # inline metadata inline "\n\n" # post content (string (item "content") "\n\n") # embedded media media)) (file/close fout)) (defn download-bookmarks [] "Downloads the bookmarks and saves them to a file." (while (>= (length urls) 0) (def url (array/shift urls)) (print (string "Downloading " url)) (def response (r/get url {"Authorization" (string "Bearer " apptoken)} {})) # Pagination (def next-link (get-link response "next")) (if (and next-link (not (has-value? done next-link))) (array/insert urls 0 next-link)) (def prev-link (get-link response "prev")) (if (and prev-link (not (has-value? done prev-link))) (array/insert urls 0 prev-link)) (def bookmarks (json/decode (response :body))) # Debugging - write response to file (def fout (file/open "./response.json" :w)) (file/write fout (response :body)) # Write the bookmarks to md files (each item bookmarks (write-markdown item)) (array/push done url))) (defn main [& args] (download-bookmarks) (print "Done."))