Class: E621ExportDownloader::Export

Inherits:
Object
  • Object
show all
Extended by:
T::Generic, T::Sig
Defined in:
lib/e621_export_downloader/export.rb

Constant Summary collapse

Model = 
type_member

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(data:, client:, type:, parser:) ⇒ Export

Returns a new instance of Export.



26
27
28
29
30
31
32
 
# File 'lib/e621_export_downloader/export.rb', line 26

def initialize(data:, client:, type:, parser:)
  @data = T.let(data, APIExportData)
  @client = T.let(client, Client)
  @type = T.let(type, Types)
  @parser = T.let(parser, Client::Options::Parser)
  @downloaded = T.let(nil, T.nilable(T::Boolean))
end

Instance Attribute Details

#client ⇒ Object

Returns the value of attribute client.



17
18
19
 
# File 'lib/e621_export_downloader/export.rb', line 17

def client
  @client
end

#data ⇒ Object

Returns the value of attribute data.



14
15
16
 
# File 'lib/e621_export_downloader/export.rb', line 14

def data
  @data
end

#downloaded ⇒ Object

Returns the value of attribute downloaded.



23
24
25
 
# File 'lib/e621_export_downloader/export.rb', line 23

def downloaded
  @downloaded
end

#type ⇒ Object

Returns the value of attribute type.



20
21
22
 
# File 'lib/e621_export_downloader/export.rb', line 20

def type
  @type
end

Instance Method Details

#delete ⇒ Object 



35
36
37
38
39
40
41
 
# File 'lib/e621_export_downloader/export.rb', line 35

def delete
  return false unless check_downloaded
  client.debug("deleting cached export", header: ["export:#{type.serialize}"])
  FileUtils.rm(file_path)
  @downloaded = false
  true
end

#download ⇒ Object

Raises:



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/e621_export_downloader/export.rb', line 44

def download
  raise(ResolveError, "Export #{type.serialize} does not exist") unless exists?
  if check_downloaded
    client.debug("using cached export", header: ["export:#{type.serialize}"])
    return file_path
  end

  client.debug("downloading export", header: ["export:#{type.serialize}"])

  FileUtils.mkdir_p(Constants::TEMP_DIR)
  File.open(file_path, "wb") do |file|
    inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)

    res = client.connection.get(data.url) do |req|
      req.options.on_data = proc do |chunk, _total|
        decompressed = inflater.inflate(chunk)
        file.write(decompressed) if decompressed && !decompressed.empty?
      end
    end

    file.write(inflater.finish)
    inflater.close
    file.close

    unless res.success?
      raise(ResolveError, "Failed to download export #{type.serialize}: #{res.status} #{res.reason_phrase}")
    end
  rescue # rubocop:disable Style/RescueStandardError
    FileUtils.rm_f(file_path)
    raise
  end

  @downloaded = true
  client.debug("download complete: #{file_path}", header: ["export:#{type.serialize}"])
  file_path
end

#exists? ⇒ Boolean

Returns:

  • (Boolean) 


82
83
84
85
86
 
# File 'lib/e621_export_downloader/export.rb', line 82

def exists?
  result = client.connection.head(data.url).success?
  client.debug("checked export existence: #{result}", header: ["export:#{type.serialize}"])
  result
end

#read(&block) ⇒ Object 



89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/e621_export_downloader/export.rb', line 89

def read(&block)
  download unless check_downloaded
  client.debug("reading export", header: ["export:#{type.serialize}"])
  total = line_count
  CSV.foreach(file_path, headers: true, converters: ->(f) { f.nil? ? "" : f }) do |row|
    args = [@parser.call(T.cast(row, CSV::Row).to_hash), total]
    args = args.slice(0, block.arity) if block.arity != -1
    block.call(*T.unsafe(args))
  end
  client.debug("finished reading export", header: ["export:#{type.serialize}"])
  delete unless client.options.cache
end

#read_all ⇒ Object 



103
104
105
106
107
108
109
110
 
# File 'lib/e621_export_downloader/export.rb', line 103

def read_all
  client.debug("reading all records", header: ["export:#{type.serialize}"])
  results = []
  read do |record|
    results << record
  end
  results
end