Yay it works

This commit is contained in:
Simon Cambier 2024-06-22 23:42:37 +02:00
parent 6eabd05968
commit 89cad9b78c
2 changed files with 92 additions and 26 deletions

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
build build
jpm_tree jpm_tree
response.json response.json
bookmarks

View File

@ -1,47 +1,112 @@
(import httprequest :as r) (import httprequest :as r)
(import spork/json :as json) (import spork/json :as json)
(def url "https://hachyderm.io/api/v1/bookmarks") (def apptoken "REDACTED")
(def auth "") (def urls @["https://hachyderm.io/api/v1/bookmarks"])
(def done @[])
(defn array/shift [arr]
"Removes the first element from the array."
(if (empty? arr) (break nil))
(def head (in arr 0))
(array/remove arr 0)
head)
(defn get-link [response rel] (defn get-link [response rel]
"Gets the link with the specified relation from the response. "Gets the link with the specified relation from the response.
e.g. (get-link response \"next\")" e.g. (get-link response \"next\")"
# Get the links from the headers # Get the links from the headers
(def links (string/split "," ((response :headers) "Link"))) (def link-header ((response :headers) "Link"))
(if (= link-header nil) (break nil))
(def links (string/split "," link-header))
# Find the link with the specified relation # Find the link with the specified relation
(def link (find |(not= (string/find rel $) nil) links)) (def link (find |(not= (string/find rel $) nil) links))
# Extract the url from the link string (as an array of 1 element) # Extract the url from the link string (as an array of 1 element)
(def url (peg/match ~(sequence (any " ") "<" (capture (some (if-not ">" 1))) ">") link)) (def url (peg/match ~(sequence (any " ") "<" (capture (some (if-not ">" 1))) ">") link))
(if url (in url 0) nil)) (if url (in url 0) nil))
(defn download-bookmarks []
"Downloads the bookmarks and saves them to a file." (defn download-media [item]
"Downloads the media from the item."
(def files @[])
(def media (item "media_attachments"))
(each m media
(def url (m "url"))
(def filename (array/pop (string/split "/" url)))
(def fullpath (string "./bookmarks/attachments/" filename))
(array/push files filename)
# Check if the file already exists
(def existing (file/open fullpath :r))
(if existing (file/close existing))
# If it doesn't exist, download it
(if (not existing)
(do
(print (string "Downloading " url))
(def response (def response
(r/get url {"Authorization" (string "Bearer " auth)} {})) (r/get url {"Authorization" (string "Bearer " apptoken)} {}))
(def next-link (get-link response "next")) (def fout (file/open fullpath :w))
(def prev-link (get-link response "prev")) (file/write fout (response :body))
(pp next-link) (file/close fout))))
(pp prev-link) files)
(def bookmarks (json/decode (response :body)))
(def fout (file/open "./response.json" :w))
(file/write fout (json/encode bookmarks)))
(defn write-markdown [item] (defn write-markdown [item]
(def files (download-media item))
(def id (item "id")) (def id (item "id"))
(def fout (file/open (string "./" id ".md") :w)) (def fout (file/open (string "./bookmarks/" id ".md") :w))
(file/write fout (item "content")) (def frontmatter (string "---\n"
"url: " (item "url") "\n"
"created: " (item "created_at") "\n"
"user: " ((item "account") "url") "\n"
"---\n"))
(def media (string/join
(map (fn [arg] (string "![[" arg "]]")) files) " "))
(file/write fout
(string
# metadata
frontmatter
# content
(string "\n" (item "content") "\n\n")
# embedded media
media))
(file/close fout)) (file/close fout))
(defn write-files []
"Writes the bookmarks to files." (defn download-bookmarks []
(def fin (file/open "./response.json" :r)) "Downloads the bookmarks and saves them to a file."
(print (file/read fin :all)) (while (>= (length urls) 0)
(def bookmarks (json/decode (file/read fin :all))) (def url (array/shift urls))
(print (string "Downloading " url))
(def response
(r/get url {"Authorization" (string "Bearer " apptoken)} {}))
# Pagination
(def next-link (get-link response "next"))
(if (and next-link (not (has-value? done next-link)))
(array/insert urls 0 next-link))
(def prev-link (get-link response "prev"))
(if (and prev-link (not (has-value? done prev-link)))
(array/insert urls 0 prev-link))
(def bookmarks (json/decode (response :body)))
# Debugging - write response to file
# (def fout (file/open "./response.json" :w))
# (file/write fout (response :body))
# Write the bookmarks to md files
(each item bookmarks (each item bookmarks
(write-markdown item))) (write-markdown item))
(array/push done url)))
(defn main [& args] (defn main [& args]
(download-bookmarks) (download-bookmarks)
# (write-files) (print "Done."))
(print "ok"))