Module: KineticSdk::Utils::KineticExportUtils

Included in:
Core, Task
Defined in:
lib/kinetic_sdk/utils/kinetic-export-utils.rb

Overview

The KineticExportUtils module provides methods to simplify exporting objects and writing to the local file system.

Instance Method Summary collapse

Instance Method Details

#get_variable_property(export_shape, object_path) ⇒ String

Fetches the variable property of a given path (ie slug, name, etc)

Parameters:

  • export_shape (Hash) —

    the directory and file structure of how the data should be written

  • object_path (String) —

    the path in which to get the variable property from

Returns:

  • (String) —

    the value of the variable property, or nil if it doesn't exist



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/kinetic_sdk/utils/kinetic-export-utils.rb', line 67

def get_variable_property(export_shape, object_path)
  # Prepare the metadata
  object_path_segments = object_path.split('.')
  pointer = export_shape
  # Walk the object path
  object_path_segments.each do |object_path_segment|
    # If the object path corresponds to a "variable"
    if pointer.has_key?(:variable)
      pointer = pointer[pointer[:variable]]
    # If the object path corresponds to a static property
    else
      pointer = pointer[object_path_segment]
    end
    # If the pointer is null (indicating that the property should not be extracted)
    break if pointer.nil?
  end
  # Return the result
  pointer.nil? ? nil : pointer[:variable]
end

#prepare_shape(*args) ⇒ Hash

Builds the tree structure of the export including the arguments

Parameters:

  • args (Array) —

    list of property names to include in the export: (i.e. slug, name, etc)

Returns:

  • (Hash) —

    struture of the shape to be exported



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/kinetic_sdk/utils/kinetic-export-utils.rb', line 13

def prepare_shape(*args)
  shape = {}
  args.each do |arg|
    segments = arg.split('.')
    pointer = shape
    segments.each do |segment|
      if segment =~ /\{(.*)\}/
        pointer[:variable] ||= $1
        pointer[$1] ||= {}
        pointer = pointer[$1]
      else
        pointer[segment] ||= {}
        pointer = pointer[segment]
      end
    end
  end
  shape
end

#process_export(core_path, export_shape, object, object_path = '', executor: nil) ⇒ Object

Processes and writes data exported from the core service

Parameters:

  • core_path (String) —

    the root folder path to write the data to

  • export_shape (Hash) —

    the directory and file structure of how the data should be written

  • object (Hash) —

    the object being processed

  • object_path (String) (defaults to: '') —

    the path of the object being processed (used recursively)

Returns:

  • nil



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/kinetic_sdk/utils/kinetic-export-utils.rb', line 120

def process_export(core_path, export_shape, object, object_path='', executor: nil)
  # Prepare metadata (same as before)
  child_objects = {}
  file_contents = {}
  
  if object.kind_of?(Array)
    file_contents = object
  else
    object.each do |key, value|
      child_object_path = object_path.empty? ? key : "#{object_path}.#{key}"

      if should_extract(child_object_path, export_shape)
        if value.kind_of?(Array)
          variable = get_variable_property(export_shape, child_object_path)
          if variable.nil?
            child_objects[child_object_path] = value
          else
            value.each do |item|
              child_objects["#{child_object_path}.#{item[variable]}"] = item
            end
          end
        else
          child_objects[child_object_path] = value
        end
      else
        file_contents[key] = value
      end
    end
  end

  if object_path != '' && !file_contents.empty?
    filename = "#{core_path}/#{object_path.gsub('/', '').gsub('\\', '').gsub('.', '/').gsub(/::/, '-').gsub(/[^ a-zA-Z0-9_\-\/]/, '')}.json"
    write_object_to_file(filename, file_contents)
  end

  # Thread at key levels: root, kapps, and forms
  should_thread = executor && child_objects.size > 1 && should_parallelize?(object_path)
  
  if should_thread
    puts "  -> Spawning #{child_objects.size} threads for: #{object_path}"
    futures = child_objects.map do |key, child_object|
      Concurrent::Future.execute(executor: executor) do
        puts "    [Thread #{Thread.current.object_id}] Starting: #{key}"
        process_export(core_path, export_shape, child_object, key, executor: executor)
        puts "    [Thread #{Thread.current.object_id}] Finished: #{key}"
      end
    end
    futures.each(&:value)
  else
    child_objects.each do |key, child_object|
      process_export(core_path, export_shape, child_object, key, executor: executor)
    end
  end
end

#process_export_old(core_path, export_shape, object, object_path = '', executor: nil) ⇒ Object 



196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
 
# File 'lib/kinetic_sdk/utils/kinetic-export-utils.rb', line 196

def process_export_old(core_path, export_shape, object, object_path='', executor: nil)
  # Prepare metadata
  child_objects = {}
  file_contents = {}
  if object.kind_of?(Array)
    file_contents = object
  else
    # For each of the object properties
    object.each do |key, value|
      # Build child object path
      child_object_path = object_path.empty? ? key : "#{object_path}.#{key}"

      # If the property should be extracted into its own folder/file
      if should_extract(child_object_path, export_shape)
        if value.kind_of?(Array)
          variable = get_variable_property(export_shape, child_object_path)
          if variable.nil?
            child_objects[child_object_path] = value
          else
            value.each do |item|
              child_objects["#{child_object_path}.#{item[variable]}"] = item
            end
          end
        else
          child_objects[child_object_path] = value
        end
      # If the property does not need to be extracted
      else
        # Add the property to the file contents
        file_contents[key] = value
      end
    end
  end

  # If this is not the "root" object
  if object_path != '' && !file_contents.empty?
    # Replace all `/` with ``
    # Replace all `\` with ``
    # Replace all `.` with `/`
    # Replace all `::` with `-` (this ensures nested Teams/Categories maintain a separator)
    # Replace all non-slug characters with ``
    filename = "#{core_path}/#{object_path.gsub('/', '').gsub('/\\', '').gsub('.', '/').gsub(/::/, '-').gsub(/[^ a-zA-Z0-9_\-\/]/, '')}.json"
    # Write the file_contents based upon the
    write_object_to_file(filename, file_contents)
  end

  if executor && child_objects.size > 0
    futures = child_objects.map do |key, child_object|
      Concurrent::Future.execute(executor: executor) do
        # Recursive calls don't use threading
        process_export(core_path, export_shape, child_object, key)
      end
    end
    futures.each do |future|
      future.value # This will re-raise any exceptions that occurred
    end
  else
    child_objects.each do |key, child_object|
      process_export(core_path, export_shape, child_object, key)
    end
  end
end

#should_extract(export_path, export_shape) ⇒ Boolean

Determines if the current path should be extracted further

Parameters:

  • export_path (String) —

    the path in which to determine if further extraction should occur

  • export_shape (Hash) —

    the directory and file structure of how the data should be written

Returns:

  • (Boolean) —

    true if the path should be extracted further, otherwise false



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/kinetic_sdk/utils/kinetic-export-utils.rb', line 37

def should_extract(export_path, export_shape)
  # Prepare the metadata
  export_path_segments = export_path.split('.')
  pointer = export_shape
  result = true
  # Walk the export path
  export_path_segments.each do |export_path_segment|
    # If the object path corresponds to a "variable"
    if pointer.has_key?(:variable)
      pointer = pointer[pointer[:variable]]
    # If the object path corresponds to a static property
    else
      pointer = pointer[export_path_segment]
    end
    # If the pointer is null (indicating that the property should not be extracted)
    if pointer.nil?
      # Break out of the loop
      result = false
      break
    end
  end
  # Return the result
  result
end

#should_parallelize?(object_path) ⇒ Boolean

Returns:

  • (Boolean) 


175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
 
# File 'lib/kinetic_sdk/utils/kinetic-export-utils.rb', line 175

def should_parallelize?(object_path)
  # Thread at these specific levels
  parallelize_patterns = [
    /^$/,                           # Root level
    /^space\.kapps$/,               # All kapps
    /^space\.kapps\.[^.]+$/,        # Within each kapp (forms, webhooks, etc)
    /^space\.kapps\.[^.]+\.forms$/, # All forms within a kapp
    /^space\.teams$/,               # All teams
    /^space\.bridges$/,             # All bridges
    /^space\.webApis$/,             # All webApis
    /^space\.webhooks$/,            # All webhooks
  ]
  
    matches = parallelize_patterns.any? { |pattern| object_path =~ pattern }

    # DEBUG
    puts "  Checking parallelize for: '#{object_path}' -> #{matches}"
    
    matches
end

#write_object_to_file(filename, file_contents) ⇒ Object

Creates directory structure and writes file

Parameters:

  • filename (String) —

    the full path and name of the file to be written

  • file_contents (String) —

    the content of the file to be written

Returns:

  • nil



92
93
94
95
96
97
98
 
# File 'lib/kinetic_sdk/utils/kinetic-export-utils.rb', line 92

def write_object_to_file(filename, file_contents)
  # Create Folder if not exists
  dir_path = File.dirname(filename)
  FileUtils.mkdir_p(dir_path, :mode => 0700)
  # Write File
  File.open(filename, 'w') { |file| file.write(JSON.pretty_generate(file_contents)) }
end

#write_object_to_file_threaded(filename, file_contents, mutex = nil) ⇒ Object 



100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/kinetic_sdk/utils/kinetic-export-utils.rb', line 100

def write_object_to_file_threaded(filename, file_contents, mutex=nil)
  # Create Folder if not exists
  dir_path = File.dirname(filename)
  if mutex
    mutex.synchronize { FileUtils.mkdir_p(dir_path, :mode => 0700) }
  else
    FileUtils.mkdir_p(dir_path, :mode => 0700)
  end
  # Write File
  File.open(filename, 'w') { |file| file.write(JSON.pretty_generate(file_contents)) }
end