Class: E621ExportDownloader::Export

Inherits:
Object
  • Object
show all
Extended by:
T::Generic, T::Sig
Defined in:
lib/e621_export_downloader/export.rb

Constant Summary collapse

Model = 
type_member

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(data:, client:, type:, parser:) ⇒ Export

Returns a new instance of Export.



30
31
32
33
34
35
36
 
# File 'lib/e621_export_downloader/export.rb', line 30

def initialize(data:, client:, type:, parser:)
  @data = T.let(data, APIExportData)
  @client = T.let(client, Client)
  @type = T.let(type, Types)
  @parser = T.let(parser, Client::Options::Parser)
  @downloaded = T.let(nil, T.nilable(T::Boolean))
end

Instance Attribute Details

#client ⇒ Object

Returns the value of attribute client.



21
22
23
 
# File 'lib/e621_export_downloader/export.rb', line 21

def client
  @client
end

#data ⇒ Object

Returns the value of attribute data.



18
19
20
 
# File 'lib/e621_export_downloader/export.rb', line 18

def data
  @data
end

#downloaded ⇒ Object

Returns the value of attribute downloaded.



27
28
29
 
# File 'lib/e621_export_downloader/export.rb', line 27

def downloaded
  @downloaded
end

#type ⇒ Object

Returns the value of attribute type.



24
25
26
 
# File 'lib/e621_export_downloader/export.rb', line 24

def type
  @type
end

Instance Method Details

#delete ⇒ Object 



39
40
41
42
43
44
45
 
# File 'lib/e621_export_downloader/export.rb', line 39

def delete
  return false unless check_downloaded
  client.debug("deleting cached export", header: ["export:#{type.serialize}"])
  FileUtils.rm(file_path)
  @downloaded = false
  true
end

#download(&block) ⇒ Object

Raises:



48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
 
# File 'lib/e621_export_downloader/export.rb', line 48

def download(&block)
  raise(ResolveError, "Export #{type.serialize} does not exist") unless exists?
  if check_downloaded
    client.debug("using cached export", header: ["export:#{type.serialize}"])
    return file_path
  end

  client.debug("downloading export", header: ["export:#{type.serialize}"])

  FileUtils.mkdir_p(Constants::TEMP_DIR)
  digest = T.let(client.options.validate_checksum ? Digest::MD5.new : nil, T.nilable(Digest::MD5))
  File.open(file_path, "wb") do |file|
    inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)

    res = client.connection.get(data.url) do |req|
      req.options.on_data = proc do |chunk, received|
        digest&.update(chunk)
        decompressed = inflater.inflate(chunk)
        file.write(decompressed) if decompressed && !decompressed.empty?
        block.call(received, data.file_size) if block
      end
    end

    file.write(inflater.finish)
    inflater.close
    file.close

    unless res.success?
      raise(ResolveError, "Failed to download export #{type.serialize}: #{res.status} #{res.reason_phrase}")
    end

    if digest && digest.hexdigest != data.checksum
      raise(ResolveError, "Checksum mismatch for export #{type.serialize}")
    end
  rescue # rubocop:disable Style/RescueStandardError
    FileUtils.rm_f(file_path)
    raise
  end

  @downloaded = true
  client.debug("download complete: #{file_path}", header: ["export:#{type.serialize}"])
  file_path
end

#exists? ⇒ Boolean

Returns:

  • (Boolean) 


93
94
95
96
97
 
# File 'lib/e621_export_downloader/export.rb', line 93

def exists?
  result = client.connection.head(data.url).success?
  client.debug("checked export existence: #{result}", header: ["export:#{type.serialize}"])
  result
end

#read(&block) ⇒ Object 



100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/e621_export_downloader/export.rb', line 100

def read(&block)
  download unless check_downloaded
  client.debug("reading export", header: ["export:#{type.serialize}"])
  total = line_count
  CSV.foreach(file_path, headers: true, converters: ->(f) { f.nil? ? "" : f }) do |row|
    args = [@parser.call(T.cast(row, CSV::Row).to_hash), total]
    args = args.slice(0, block.arity) if block.arity != -1
    block.call(*T.unsafe(args))
  end
  client.debug("finished reading export", header: ["export:#{type.serialize}"])
  delete unless client.options.cache
end

#read_all ⇒ Object 



114
115
116
117
118
119
120
121
 
# File 'lib/e621_export_downloader/export.rb', line 114

def read_all
  client.debug("reading all records", header: ["export:#{type.serialize}"])
  results = []
  read do |record|
    results << record
  end
  results
end