Class: GeoCombine::Harvester

Inherits:
Object
  • Object
show all
Defined in:
lib/geo_combine/harvester.rb

Overview

Harvests Geoblacklight documents from OpenGeoMetadata for indexing

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(ogm_path:, skip_repos: ENV.fetch('OGM_SKIP_REPOS', '').split(',').map(&:strip), skip_restricted: ENV.fetch('OGM_SKIP_RESTRICTED', 'true') == 'true', schema_version: ENV.fetch('SCHEMA_VERSION', 'Aardvark'), logger: GeoCombine::Logger.logger) ⇒ Harvester

Initialize a new harvester

Parameters:

  • ogm_path (String) —

    path to the directory where repositories will be cloned

  • skip_repos (Array<String>) (defaults to: ENV.fetch('OGM_SKIP_REPOS', '').split(',').map(&:strip)) —

    list of repository names to skip

  • skip_restricted (Boolean) (defaults to: ENV.fetch('OGM_SKIP_RESTRICTED', 'true') == 'true') —

    whether to skip indexing restricted documents

  • schema_version (String) (defaults to: ENV.fetch('SCHEMA_VERSION', 'Aardvark')) —

    schema version to filter repositories by

  • logger (Logger) (defaults to: GeoCombine::Logger.logger) —

    logger to use for logging messages



25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/geo_combine/harvester.rb', line 25

def initialize(
  ogm_path:,
  skip_repos: ENV.fetch('OGM_SKIP_REPOS', '').split(',').map(&:strip),
  skip_restricted: ENV.fetch('OGM_SKIP_RESTRICTED', 'true') == 'true',
  schema_version: ENV.fetch('SCHEMA_VERSION', 'Aardvark'),
  logger: GeoCombine::Logger.logger
)
  @ogm_path = ogm_path
  @schema_version = schema_version
  @skip_repos = skip_repos
  @skip_restricted = skip_restricted
  @logger = logger
end

Instance Attribute Details

#ogm_path ⇒ Object (readonly)

Returns the value of attribute ogm_path.



12
13
14
 
# File 'lib/geo_combine/harvester.rb', line 12

def ogm_path
  @ogm_path
end

#schema_version ⇒ Object (readonly)

Returns the value of attribute schema_version.



12
13
14
 
# File 'lib/geo_combine/harvester.rb', line 12

def schema_version
  @schema_version
end

Class Method Details

.ogm_api_uri ⇒ Object

GitHub API endpoint for OpenGeoMetadata repositories



15
16
17
 
# File 'lib/geo_combine/harvester.rb', line 15

def self.ogm_api_uri
  URI('https://api.github.com/orgs/opengeometadata/repos?per_page=1000')
end

Instance Method Details

#clone(repo) ⇒ Object

Clone a repository via git Return the name of the repository cloned, or nil if skipped



104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
 
# File 'lib/geo_combine/harvester.rb', line 104

def clone(repo)
  repo_path = File.join(@ogm_path, repo)
  repo_info = repository_info(repo)
  repo_url = "https://github.com/OpenGeoMetadata/#{repo}.git"
  repo_schemas = Array(repo_info.dig('custom_properties', 'supported_schemas'))

  # Skip if exists, archived, empty, or different schema
  return @logger.warn "skipping clone to #{repo_path}; directory exists" if File.directory? repo_path
  return @logger.warn "repository is archived: #{repo_url}" if repo_info['archived']
  return @logger.warn "repository is empty: #{repo_url}" if repo_info['size'].zero?
  unless repo_schemas.include? @schema_version
    return @logger.warn "repository #{repo_url} clone to #{repo_path}; repository properties don't include schema version #{@schema_version} (found #{repo_schemas.join(', ')})"
  end

  Git.clone(repo_url, nil, path: ogm_path, depth: 1)
  @logger.info "cloned #{repo_url} to #{repo_path}"
  repo
end

#clone_all ⇒ Object

Clone all repositories via git Return the names of repositories cloned.



125
126
127
128
129
 
# File 'lib/geo_combine/harvester.rb', line 125

def clone_all
  cloned = repositories.map(&method(:clone)).compact
  @logger.info "cloned #{cloned.size} repositories"
  cloned
end

#docs_to_index ⇒ Object

Enumerable of docs to index, for passing to an indexer rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/geo_combine/harvester.rb', line 41

def docs_to_index
  return to_enum(:docs_to_index) unless block_given?

  @logger.info "loading documents from #{ogm_path}"
  Find.find(@ogm_path) do |path|
    # skip non-json and layers.json files
    if File.basename(path) == 'layers.json' || !File.basename(path).end_with?('.json')
      @logger.debug "skipping #{path}; not a geoblacklight JSON document"
      next
    end

    doc = JSON.parse(File.read(path))
    [doc].flatten.each do |record|
      record_schema = record['gbl_mdVersion_s'] || record['geoblacklight_version']
      record_id = record['layer_slug_s'] || record['dc_identifier_s']
      record_rights = record['dct_accessRights_s'] || record['dc_rights_s']

      # skip indexing if no identifiable schema version
      unless record_schema
        @logger.debug "skipping #{record_id || path}; no schema version declared in record"
        next
      end

      # skip indexing if this record has a different schema version than what we want
      if record_schema != @schema_version
        @logger.debug "skipping #{record_id}; schema version #{record_schema} doesn't match #{@schema_version}"
        next
      end

      # skip indexing if this record is restricted and we want to skip restricted records
      if @skip_restricted && record_rights == 'Restricted'
        @logger.debug "skipping #{record_id}; access rights are restricted"
        next
      end

      @logger.debug "found record #{record_id} at #{path}"
      yield record, path
    end
  end
end

#pull(repo) ⇒ Object

Update a repository via git If the repository doesn't exist, clone it.



85
86
87
88
89
90
91
92
 
# File 'lib/geo_combine/harvester.rb', line 85

def pull(repo)
  repo_path = File.join(@ogm_path, repo)
  clone(repo) unless File.directory? repo_path

  Git.open(repo_path).pull
  @logger.info "updated #{repo}"
  repo
end

#pull_all ⇒ Object

Update all repositories Return the names of repositories updated



96
97
98
99
100
 
# File 'lib/geo_combine/harvester.rb', line 96

def pull_all
  updated = repositories.map(&method(:pull)).compact
  @logger.info "updated #{updated.size} repositories"
  updated
end