Class: ZMediumFetcher

Inherits:
Object
  • Object
show all
Defined in:
lib/ZMediumFetcher.rb

Defined Under Namespace

Classes: Progress

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ ZMediumFetcher

Returns a new instance of ZMediumFetcher.



72
73
74
75
76
77
78
 
# File 'lib/ZMediumFetcher.rb', line 72

def initialize
    @progress = Progress.new()
    @usersPostURLs = nil
    @isForJekyll = false
    @stdoutIO = nil
    @stdoutMode = false
end

Instance Attribute Details

#isForJekyll ⇒ Object

Returns the value of attribute isForJekyll.



32
33
34
 
# File 'lib/ZMediumFetcher.rb', line 32

def isForJekyll
  @isForJekyll
end

#progress ⇒ Object

Returns the value of attribute progress.



32
33
34
 
# File 'lib/ZMediumFetcher.rb', line 32

def progress
  @progress
end

#stdoutIO ⇒ Object

Returns the value of attribute stdoutIO.



32
33
34
 
# File 'lib/ZMediumFetcher.rb', line 32

def stdoutIO
  @stdoutIO
end

#stdoutMode ⇒ Object

Returns the value of attribute stdoutMode.



32
33
34
 
# File 'lib/ZMediumFetcher.rb', line 32

def stdoutMode
  @stdoutMode
end

#usersPostURLs ⇒ Object

Returns the value of attribute usersPostURLs.



32
33
34
 
# File 'lib/ZMediumFetcher.rb', line 32

def usersPostURLs
  @usersPostURLs
end

Class Method Details

.decodePathPreservingSpaces(url) ⇒ Object

Encode-then-decode helper that preserves spaces as %20. Replaces the previous module-level URI.decode monkey patch.



36
37
38
39
 
# File 'lib/ZMediumFetcher.rb', line 36

def self.decodePathPreservingSpaces(url)
    return "" if url.nil?
    URI.decode_www_form_component(url).gsub(" ", "%20")
end

Instance Method Details

#buildParser(imagePathPolicy, skipImages: false) ⇒ Object 



80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/ZMediumFetcher.rb', line 80

def buildParser(imagePathPolicy, skipImages: false)
    h1Parser = H1Parser.new()
    h2Parser = H2Parser.new()
    h3Parser = H3Parser.new()
    h4Parser = H4Parser.new()
    ppParser = PParser.new()
    uliParser = ULIParser.new()
    oliParser = OLIParser.new()
    mixtapeembedParser = MIXTAPEEMBEDParser.new(isForJekyll)
    pqParser = PQParser.new()
    iframeParser = IframeParser.new(isForJekyll, skipImages: skipImages)
    iframeParser.pathPolicy = imagePathPolicy
    imgParser = IMGParser.new(isForJekyll, skipImages: skipImages)
    imgParser.pathPolicy = imagePathPolicy
    bqParser = BQParser.new()
    preParser = PREParser.new(isForJekyll)
    codeBlockParser = CodeBlockParser.new(isForJekyll)
    fallbackParser = FallbackParser.new()

    chain = [
        h1Parser, h2Parser, h3Parser, h4Parser, ppParser, uliParser, oliParser,
        mixtapeembedParser, pqParser, iframeParser, imgParser, bqParser,
        preParser, codeBlockParser, fallbackParser
    ]
    chain.each_cons(2) { |a, b| a.setNext(b) }

    chain.first
end

#downloadPost(postURL, pathPolicy, isPin) ⇒ Object 



109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/ZMediumFetcher.rb', line 109

def downloadPost(postURL, pathPolicy, isPin)
    return downloadPostToStdout(postURL, isPin) if stdoutMode

    postID = Post.getPostIDFromPostURLString(postURL)

    if isForJekyll
        postPath = postID # use only post id is more friendly for url seo
        postPathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath("_posts/zmediumtomarkdown"), pathPolicy.getRelativePath("_posts/zmediumtomarkdown"))
        imagePathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath("assets"), "assets")
    else
        postPath = Post.getPostPathFromPostURLString(postURL)
        postPathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath("zmediumtomarkdown"), pathPolicy.getRelativePath("zmediumtomarkdown"))
        imagePathPolicy = PathPolicy.new(postPathPolicy.getAbsolutePath("assets"), "assets")
    end

    progress.postPath = ZMediumFetcher.decodePathPreservingSpaces(postPath)
    progress.message = "Downloading Post..."
    progress.printLog()

    postInfo = Post.parsePostInfo(postID, imagePathPolicy)
    raise "Error: Post info not found! PostURL: #{postURL}" if postInfo.nil?

    contentInfo = Post.fetchPostParagraphs(postID)
    raise "Error: Paragraph Content not found! PostURL: #{postURL}" if contentInfo.nil?

    isLockedPreviewOnly = contentInfo&.dig("isLockedPreviewOnly")

    sourceParagraphs = contentInfo&.dig("bodyModel", "paragraphs")
    raise "Error: Paragraph not found! PostURL: #{postURL}" if sourceParagraphs.nil?

    progress.message = "Formatting Data..."
    progress.printLog()

    paragraphs = preprocessParagraphs(sourceParagraphs, postID)

    startParser = buildParser(imagePathPolicy)

    progress.totalPostParagraphsLength = paragraphs.length
    progress.currentPostParagraphIndex = 0
    progress.message = "Converting Post..."
    progress.printLog()

    postWithDatePath = "#{postInfo.firstPublishedAt.strftime("%Y-%m-%d")}-#{postPath}"
    absolutePath = ZMediumFetcher.decodePathPreservingSpaces(postPathPolicy.getAbsolutePath("#{postWithDatePath}")) + ".md"

    existingMeta = readExistingFrontMatter(absolutePath)

    if shouldSkipExistingPost?(existingMeta, postInfo, isPin, isLockedPreviewOnly)
        # Already downloaded and nothing has changed!, Skip!
        progress.currentPostParagraphIndex = paragraphs.length
        progress.message = "Skip, Post already downloaded and nothing has changed!"
        progress.printLog()
    else
        Helper.createDirIfNotExist(postPathPolicy.getAbsolutePath(nil))
        File.open(absolutePath, "w+") do |file|
            writePost(file, paragraphs, postInfo, isLockedPreviewOnly, postURL, isPin, startParser)
        end
        FileUtils.touch absolutePath, :mtime => postInfo.latestPublishedAt

        progress.message = if isLockedPreviewOnly
                               paywallMessage
                           else
                               "Post Successfully Downloaded!"
                           end
        progress.printLog()
    end

    progress.postPath = nil
end

#downloadPostsByUsername(username, pathPolicy, limit: nil) ⇒ Object 



224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
 
# File 'lib/ZMediumFetcher.rb', line 224

def downloadPostsByUsername(username, pathPolicy, limit: nil)
    progress.username = username
    progress.message = "Fetching posts..."
    progress.printLog()

    userID = User.convertToUserIDFromUsername(username)
    raise "Medium's Username:#{username} not found!" if userID.nil?

    postURLS = []
    nextID = nil
    loop do
        postPageInfo = User.fetchUserPosts(userID, nextID)
        postURLS.concat(postPageInfo["postURLs"])
        nextID = postPageInfo["nextID"]
        break if nextID.nil?
        break if !limit.nil? && postURLS.length >= limit
    end

    postURLS = postURLS.first(limit) unless limit.nil?

    @usersPostURLs = postURLS.map { |post| post["url"] }

    progress.totalPostsLength = postURLS.length
    progress.currentPostIndex = 0
    progress.message = "Downloading posts..."
    progress.printLog()

    downloadPathPolicy = nil
    unless stdoutMode
        downloadPathPolicy = if isForJekyll
                                 pathPolicy
                             else
                                 PathPolicy.new(pathPolicy.getAbsolutePath("users/#{username}"), pathPolicy.getRelativePath("users/#{username}"))
                             end
    end

    postURLS.each_with_index do |postURL, idx|
        begin
            downloadPost(postURL["url"], downloadPathPolicy, postURL["pin"])
            if stdoutMode && idx < postURLS.length - 1
                stdoutIO.puts "\n\n---\n\n"
            end
        rescue => e
            if stdoutMode
                warn e
            else
                puts e
            end
        end

        progress.currentPostIndex = idx + 1
        progress.message = "Downloading posts..."
        progress.printLog()
    end

    progress.message = "All posts has been downloaded!, Total posts: #{postURLS.length}"
    progress.printLog()
end

#downloadPostToStdout(postURL, isPin) ⇒ Object

Stdout fast path: render markdown directly to ‘stdoutIO` without touching the filesystem and without downloading any images. Image references stay as remote URLs on miro.medium.com (or the configured MEDIUM_HOST proxy origin when set).



183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
 
# File 'lib/ZMediumFetcher.rb', line 183

def downloadPostToStdout(postURL, isPin)
    postID = Post.getPostIDFromPostURLString(postURL)
    postPath = Post.getPostPathFromPostURLString(postURL)

    progress.postPath = ZMediumFetcher.decodePathPreservingSpaces(postPath)
    progress.message = "Rendering Post..."
    progress.printLog()

    postInfo = Post.parsePostInfo(postID, nil, skipImages: true)
    raise "Error: Post info not found! PostURL: #{postURL}" if postInfo.nil?

    contentInfo = Post.fetchPostParagraphs(postID)
    raise "Error: Paragraph Content not found! PostURL: #{postURL}" if contentInfo.nil?

    isLockedPreviewOnly = contentInfo&.dig("isLockedPreviewOnly")

    sourceParagraphs = contentInfo&.dig("bodyModel", "paragraphs")
    raise "Error: Paragraph not found! PostURL: #{postURL}" if sourceParagraphs.nil?

    progress.message = "Formatting Data..."
    progress.printLog()

    paragraphs = preprocessParagraphs(sourceParagraphs, postID)
    startParser = buildParser(nil, skipImages: true)

    progress.totalPostParagraphsLength = paragraphs.length
    progress.currentPostParagraphIndex = 0
    progress.message = "Converting Post..."
    progress.printLog()

    writePost(stdoutIO, paragraphs, postInfo, isLockedPreviewOnly, postURL, isPin, startParser)

    progress.message = if isLockedPreviewOnly
                           paywallMessage
                       else
                           "Post Successfully Rendered!"
                       end
    progress.printLog()
    progress.postPath = nil
end

#flushPreParagraphsInto(paragraphs, preTypeParagraphs) ⇒ Object

Collapses a run of PRE paragraphs already in ‘paragraphs` into a single CODE_BLOCK paragraph. The last PRE in the run is rewritten in place to hold the joined text; all earlier PREs are dropped.



485
486
487
488
489
490
491
492
493
494
 
# File 'lib/ZMediumFetcher.rb', line 485

def flushPreParagraphsInto(paragraphs, preTypeParagraphs)
    return if preTypeParagraphs.length <= 1

    last = preTypeParagraphs.last
    last.text = preTypeParagraphs.map(&:orgText).join("\n")
    last.type = CodeBlockParser.getTypeString()

    droppedNames = preTypeParagraphs[0...-1].map(&:name)
    paragraphs.reject! { |p| droppedNames.include?(p.name) }
end

#listPostsByUsername(username, limit = nil) ⇒ Object

Emits one NDJSON line per post (without bodies) to ‘stdoutIO`, used by `–list -u <username>`. Honors `limit` to short-circuit pagination and per-post metadata fetch as soon as we have enough.



286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
 
# File 'lib/ZMediumFetcher.rb', line 286

def listPostsByUsername(username, limit = nil)
    progress.username = username
    progress.message = "Fetching posts list..."
    progress.printLog()

    userID = User.convertToUserIDFromUsername(username)
    raise "Medium's Username:#{username} not found!" if userID.nil?

    postURLS = []
    nextID = nil
    loop do
        postPageInfo = User.fetchUserPosts(userID, nextID)
        postURLS.concat(postPageInfo["postURLs"])
        nextID = postPageInfo["nextID"]
        break if nextID.nil?
        break if !limit.nil? && postURLS.length >= limit
    end

    postURLS = postURLS.first(limit) unless limit.nil?

    progress.totalPostsLength = postURLS.length
    progress.currentPostIndex = 0
    progress.message = "Listing posts..."
    progress.printLog()

    postURLS.each_with_index do |entry, idx|
        url = entry["url"]
        pin = entry["pin"]

        begin
            postID = Post.getPostIDFromPostURLString(url)
            info = Post.parsePostInfo(postID, nil, skipImages: true)
            if info.nil?
                warn "Skipping #{url}: post info not found"
            else
                line = {
                    "title"             => info.title,
                    "url"               => url,
                    "creator"           => info.creator,
                    "firstPublishedAt"  => info.firstPublishedAt&.iso8601,
                    "latestPublishedAt" => info.latestPublishedAt&.iso8601,
                    "tags"              => info.tags || [],
                    "description"       => info.description,
                    "pin"               => pin == true
                }
                stdoutIO.puts JSON.generate(line)
            end
        rescue => e
            warn "Error listing post #{url}: #{e.message}"
        end

        progress.currentPostIndex = idx + 1
        progress.message = "Listing posts..."
        progress.printLog()
    end

    progress.message = "All posts listed!, Total posts: #{postURLS.length}"
    progress.printLog()
end

#paywallMessage ⇒ Object

Wording branches on whether the user supplied Medium auth cookies, so they get an actionable next step: provide cookies vs. check that cookies belong to a Medium Member account that has access to the post.



420
421
422
423
424
425
426
 
# File 'lib/ZMediumFetcher.rb', line 420

def paywallMessage
    if !defined?($cookies) || $cookies.nil? || ($cookies['sid'].to_s.empty? && $cookies['uid'].to_s.empty?)
        "This post is behind Medium's paywall. Cookies (sid / uid) are REQUIRED to download the full content — without them you only get the public preview. Setup guide: https://github.com/ZhgChgLi/ZMediumToMarkdown/blob/main/wiki/Setting-Up-Medium-Cookies-and-a-Cloudflare-Worker-Proxy.md"
    else
        "This post is behind Medium's paywall and the provided cookies don't grant access. Verify your sid / uid belong to a Medium Member account that can read this post. Cookies stay valid as long as they're being used (each successful request resets a ~2-week sliding window); they only expire after ~2 weeks of inactivity."
    end
end

#preprocessParagraphs(sourceParagraphs, postID) ⇒ Object

Walks the raw Medium paragraph dicts and produces a list of Paragraph objects, normalizing OLI numbering, inserting blank separators between list/quote runs, and merging adjacent PRE paragraphs into one CodeBlock.



442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
 
# File 'lib/ZMediumFetcher.rb', line 442

def preprocessParagraphs(sourceParagraphs, postID)
    paragraphs = []
    oliIndex = 0
    previousParagraph = nil
    preTypeParagraphs = []

    sourceParagraphs.each do |sourceParagraph|
        next if !sourceParagraph || !postID
        paragraph = Paragraph.new(sourceParagraph, postID)

        if OLIParser.isOLI(paragraph)
            oliIndex += 1
            paragraph.oliIndex = oliIndex
        else
            oliIndex = 0
        end

        if (OLIParser.isOLI(previousParagraph) && !OLIParser.isOLI(paragraph)) ||
           (ULIParser.isULI(previousParagraph) && !ULIParser.isULI(paragraph)) ||
           (BQParser.isBQ(previousParagraph) && !BQParser.isBQ(paragraph))
            paragraphs.append(Paragraph.makeBlankParagraph(postID))
        end

        if PREParser.isPRE(paragraph)
            preTypeParagraphs.append(paragraph)
        elsif PREParser.isPRE(previousParagraph)
            flushPreParagraphsInto(paragraphs, preTypeParagraphs)
            preTypeParagraphs = []
        end

        paragraphs.append(paragraph)
        previousParagraph = paragraph
    end

    # Flush any trailing PRE run (fixes posts that end in code blocks).
    flushPreParagraphsInto(paragraphs, preTypeParagraphs)

    paragraphs
end

#readExistingFrontMatter(absolutePath) ⇒ Object 



395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
 
# File 'lib/ZMediumFetcher.rb', line 395

def readExistingFrontMatter(absolutePath)
    meta = { lastModifiedAt: nil, pin: false, lockedPreviewOnly: false }
    return meta unless File.file?(absolutePath)

    lines = File.foreach(absolutePath).first(15)
    return meta unless lines.first&.start_with?("---")

    latestPublishedAtLine = lines.find { |line| line.start_with?("last_modified_at:") }
    if latestPublishedAtLine
        value = latestPublishedAtLine[/^(last_modified_at:)\s+(\S*)/, 2]
        meta[:lastModifiedAt] = Time.parse(value).to_i if value
    end

    pinLine = lines.find { |line| line.start_with?("pin:") }
    meta[:pin] = pinLine[/^(pin:)\s+(\S*)/, 2].to_s.downcase == "true" if pinLine

    lockedLine = lines.find { |line| line.start_with?("lockedPreviewOnly:") }
    meta[:lockedPreviewOnly] = lockedLine[/^(lockedPreviewOnly:)\s+(\S*)/, 2].to_s.downcase == "true" if lockedLine

    meta
end

#renderParagraph(paragraph, startParser) ⇒ Object

Runs MarkupParser (when applicable) and the parser chain on a single Paragraph, returning the rendered markdown string.



430
431
432
433
434
435
436
437
 
# File 'lib/ZMediumFetcher.rb', line 430

def renderParagraph(paragraph, startParser)
    unless CodeBlockParser.isCodeBlock(paragraph) || PREParser.isPRE(paragraph)
        markupParser = MarkupParser.new(paragraph, isForJekyll)
        markupParser.usersPostURLs = usersPostURLs
        paragraph.text = markupParser.parse()
    end
    startParser.parse(paragraph)
end

#shouldSkipExistingPost?(existingMeta, postInfo, isPin, isLockedPreviewOnly) ⇒ Boolean

Reads YAML-ish front matter from a previously-generated post and returns the fields we care about for skip-already-downloaded logic. Does the existing on-disk post still match the freshly-fetched metadata? All three signals must agree:

1. last_modified_at on disk is >= Medium's latestPublishedAt
2. isPin matches the file's pin flag
3. isLockedPreviewOnly matches the file's lockedPreviewOnly flag

Boolean signals are normalized to true/false before comparing so that ‘nil` (Medium’s GraphQL response can omit the field for non-paywalled / non-pinned posts) and ‘false` (the default written when the front- matter line is omitted by Helper.createPostInfo) are treated as equivalent — otherwise free, never-pinned posts would re-download on every run.

Returns:

  • (Boolean) 


387
388
389
390
391
392
393
 
# File 'lib/ZMediumFetcher.rb', line 387

def shouldSkipExistingPost?(existingMeta, postInfo, isPin, isLockedPreviewOnly)
    return false unless existingMeta[:lastModifiedAt]
    return false unless existingMeta[:lastModifiedAt] >= postInfo.latestPublishedAt.to_i
    return false unless (isPin == true) == existingMeta[:pin]
    return false unless (isLockedPreviewOnly == true) == existingMeta[:lockedPreviewOnly]
    true
end

#writePost(io, paragraphs, postInfo, isLockedPreviewOnly, postURL, isPin, startParser) ⇒ Object

Renders a post body to ‘io` (a File or any IO-like object). Shared by the filesystem path and the stdout path.



352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
 
# File 'lib/ZMediumFetcher.rb', line 352

def writePost(io, paragraphs, postInfo, isLockedPreviewOnly, postURL, isPin, startParser)
    postMetaInfo = Helper.createPostInfo(postInfo, isPin, isLockedPreviewOnly, isForJekyll)
    io.puts(postMetaInfo) unless postMetaInfo.nil?

    paragraphs.each_with_index do |paragraph, index|
        io.puts(renderParagraph(paragraph, startParser))

        progress.currentPostParagraphIndex = index + 1
        progress.message = "Converting Post..."
        progress.printLog()
    end

    if isLockedPreviewOnly
        viewFullPost = Helper.createViewFullPost(postURL, isForJekyll)
        io.puts(viewFullPost) unless viewFullPost.nil?
    else
        postWatermark = Helper.createWatermark(postURL, isForJekyll)
        io.puts(postWatermark) unless postWatermark.nil?
    end
end