Module: Bipm::Data::Importer::Common

Extended by:

Common

Included in:

Common

Defined in:

lib/bipm/data/importer/common.rb

Instance Method Summary

• #extract_date(date_str) ⇒ Object
• #extract_pdf(meeting, lang) ⇒ Object
• #format_message(part) ⇒ Object
• #ng_to_string(ps) ⇒ Object
• #parse_resolution(res, res_id, date, type = :cgpm, lang = "en", rec_type = nil) ⇒ Object
• #replace_centers(ps) ⇒ Object
• #replace_links(ps, res, lang) ⇒ Object

Instance Method Details

#extract_date(date_str) ⇒ Object

# File 'lib/bipm/data/importer/common.rb', line 402

def extract_date(date_str)
  return nil unless date_str

  date = date_str.strip
                 .gsub(/\s+/, " ")
                 .gsub("février", "february") # 3 first letters must match English
                 .gsub("juin", "june")
                 .gsub("avril", "april")
                 .gsub("mai", "may")
                 .gsub("juillet", "july")
                 .gsub(/ao[uû]t/, "august")
                 .gsub("décembre", "december")
                 .split(/, | to | au /) # Get last date
                 .last
  date = Date.parse(date)

  binding.pry if date <= Date.parse("0000-01-01")

  date
rescue Date::Error
  binding.pry
end

#extract_pdf(meeting, lang) ⇒ Object

# File 'lib/bipm/data/importer/common.rb', line 388

def extract_pdf(meeting, lang)
  pdfs = meeting.css('a.title-third[href*=".pdf"]')
                .map { |i| i.attr("href") }
                .map { |i| i.split("?").first }
                .select do |i|
    i.downcase.include?("-#{lang}.pdf") ||
    %w[en fr].none? { |l| i.downcase.include? "-#{l}.pdf" }
  end

  pdfs = pdfs.first if pdfs.length <= 1

  pdfs
end

#format_message(part) ⇒ Object

# File 'lib/bipm/data/importer/common.rb', line 166

def format_message(part)
  AsciiMath.asciidoc_extract_math(
    Coradoc::Input::HTML.convert(part).strip.gsub(" ", " ").gsub(" \n", "\n")
  )
rescue
  warn "Bug in Coradoc, couldn't parse the following document:"
  pp part
  warn "Please report this as an issue to https://github.com/metanorma/coradoc"
  raise
end

#ng_to_string(ps) ⇒ Object

# File 'lib/bipm/data/importer/common.rb', line 177

def ng_to_string(ps)
  ps.inner_html.encode("utf-8").gsub("\r", "").gsub(%r'</?nobr>', "")
end

#parse_resolution(res, res_id, date, type = :cgpm, lang = "en", rec_type = nil) ⇒ Object

# File 'lib/bipm/data/importer/common.rb', line 181

def parse_resolution(res, res_id, date, type = :cgpm, lang = "en", rec_type = nil)
  # Reparse the document after fixing upstream syntax
  fixed_body = res.body.gsub("<name=", "<a name=")
  fixed_body = fixed_body.force_encoding("utf-8")
  fixed_body = fixed_body.gsub("&Eacute;", "É")
  fixed_body = fixed_body.gsub("&#171;&#032;", "« ")
  fixed_body = fixed_body.gsub("&#032;&#187;", " »")
  fixed_body = fixed_body.sub(%r'<h1>(.*?)</h1>'m, "")
  supertitle = $1.strip
  fixed_body = fixed_body.sub(%r'<h2>(.*?)</h2>'m, "")
  title = $1.strip
  fixed_body = fixed_body.sub(/(="web-content">)\s*<p>\s*(<p)/, '\1\2')
  fixed_body = fixed_body.gsub(%r"<a name=\"haut\">(.*?)</a>"m, '\1')
  ng = Nokogiri::HTML(fixed_body, res.uri.to_s, "utf-8", Nokogiri::XML::ParseOptions.new.default_html.noent)

  refs = ng.css(".publication-card_reference a")

  if rec_type.end_with? "?"
    rec_type = case supertitle
      when /\AD[eé]claration/
        "statement"
      when /\AR[eé]solution/
        "resolution"
      else
        rec_type[..-2]
      end
  end

  r = {
    "dates" => [date.to_s],
    "subject" => nil,
    "type" => rec_type,
    "title" => title,
    "identifier" => res_id,
    "url" => res.uri.to_s,
    "reference" => nil,
    "reference_name" => nil,
    "reference_page" => nil,

    "approvals" => [{}],

    "considerations" => [],
    "actions" => [],
  }

  r.delete("type") unless r["type"]

  if refs.length > 0
    r["reference"] = res.uri.merge(refs.first.attr("href")).to_s.split("?").first
    name, page = refs.first.text.strip.split(/, p(?=[0-9])/)
    r["reference_name"] = name
    if page
      r["reference_page"] = page.to_i
    else
      r.delete("reference_page")
    end
  else
    r.delete("reference")
    r.delete("reference_name")
    r.delete("reference_page")
  end

  ps = ng.css("div.journal-content-article").first

  #binding.pry if ps.count != 1

  # Replace links
  Common.replace_links(ps, res, lang)

  # Replace a group of centers (> 1) with a table
  Common.replace_centers(ps)

  doc = Common.ng_to_string(ps)
  # doc = AsciiMath.html_to_asciimath(doc)

  if doc.match? DOIREGEX
    doc = doc.sub(DOIREGEX, "")
    r["doi"] = $1
  end

  parts = doc.split(/(\n(?:<p>)?<b>.*?<\/b>|\n<p><i>.*?<\/i>|<div class="bipm-lame-grey">|<h3>|<p>(?:après examen |après avoir entendu )|having noted that |decides to define |décide de définir |conformément à l'invitation|acting in accordance with|recommande que les résultats|(?:strongly |and further |)(?:considers|recommends|recommande) (?:la|that|que(?! « ))|estime que|declares<\/p>|déclare :<\/b><\/p>|<a name="_ftn\d)/)
  nparts = [parts.shift]
  while parts.length > 0
    nparts << parts.shift + parts.shift
  end

  if nparts.first =~ /(,|[mM]esures( \(C[GI]PM\))?|CGPM| \(CCTC\)| Conf[eé]rence|\[de thermométrie et calorimétrie\])[ \n]?(<\/p>)?\n?(\n|\n<p>[[:space:]]<\/p>\n)?\t?\z/
    r["approvals"].first["message"] = Common.format_message(nparts.shift)
  else
    r.delete("approvals")
  end

  prev = nil
  nparts.each do |part|
    parse = Nokogiri::HTML(part).text.strip

    if parse.start_with? /r[eé]f[eé]rence/
      next
    end

    if parse.start_with? "NOTE"
      part = part.sub("<h3>NOTE</h3>", "")
      r["notes"] = Common.format_message(part)
      next
    end

    CONSIDERATIONS.any? do |k, v|
      if parse =~ /\A#{PREFIX}#{k}\b/i
        r["considerations"] << prev = {
          "type" => v,
          "date_effective" => date.to_s,
          "message" => Common.format_message(part),
        }
      end
    end && next

    ACTIONS.any? do |k, v|
      if parse =~ /\A#{PREFIX}#{k}\b/i
        r["actions"] << prev = {
          "type" => v,
          "date_effective" => date.to_s,
          "message" => Common.format_message(part),
        }
      end
    end && next

    if parse =~ /\A(?:Appendix |Annexe |\()(\d+)/
      r["appendices"] ||= []
      r["appendices"] << prev = {
        "identifier" => $1.to_i,
        "message" => Common.format_message(part),
      }
      next
    end

    if parse =~ /\A((becquerel|gray, symbol)|\d\.|2 (Recommended|Valeurs)|«[[:space:]]?11\.)/
      prev["message"] += "\n" + Common.format_message(part)
      next
    end

    next if parse =~ /\A(|\[Cliquer ici\]|Click here|\.\.\.)\z/

    r["x-unparsed"] ||= []
    r["x-unparsed"] << parse #ReverseAdoc.convert(part).strip
  end

  %w[considerations actions].each do |type|
    map = type == "actions" ? ACTIONS : CONSIDERATIONS
    r[type] = r[type].map do |i|
      islist = false

      kk = nil

      if map.any? { |k, v| (i["message"].split("\n").first =~ /\A\s*([*_]?)(#{PREFIX}#{k})\1?(#{SUFFIX})\1?\s*\z/i) && (kk = k) }
        prefix = $2
        suffix = $3
        subject = $4

        listmarker = nil
        listitems = []
        if (i["message"].split(/(?<!\+)\n(?!\+)/).all? { |j|
          case j
          when /\A\s*[*_]?#{PREFIX}#{kk}/i
            true
          when /\A\s*\z/
            true
          when /\A(\. |\* | )(\S.*?)\z/m
            listitems << $2
            listmarker = $1 if !listmarker
            listmarker == $1
          else
            false
          end
        })
          islist = true if listitems.length >= 1
        end
      end

      if subject
        #p subject
        r["subject"] ||= []
        r["subject"] << subject
      end

      if islist
        suffix = suffix.strip
        suffix = nil if suffix == ""
        listitems.map do |li|
          i.merge "message" => [prefix, suffix, li].compact.join(" ")
        end
      else
        i
      end
    end.flatten
  end

  if r["subject"]
    r["subject"] = r["subject"].uniq.join(" and ")
  end

  # Note: we replace the previously set r['subject'].
  r["subject"] = type.to_s.upcase.gsub("-", " ")
  r["subject"] = "CCDS" if type == :cctf && supertitle.include?("CCDS")

  r
end

#replace_centers(ps) ⇒ Object

# File 'lib/bipm/data/importer/common.rb', line 125

def replace_centers(ps)
  centers = ps.css("center").to_a
  while centers.length > 0
    center = centers.first
    current = center
    mycenters = [center]
    loop do
      break unless current.next
      while Nokogiri::XML::Text === current.next
        current = current.next
        break if current.text.strip != ""
      end
      break unless current.next
      break unless current.next.name == "center"
      current = current.next
      mycenters << current
    end
    centers -= mycenters
    if mycenters.length > 1
      newtable = Nokogiri::HTML::Builder.new do |doc|
        doc.table {
          mycenters.each do |i|
            doc.tr {
              doc.td {
                doc << i.inner_html
              }
            }
          end
        }
      end.to_html
      mycenters.first.replace newtable
      mycenters[1..-1].each &:remove
    end
  end

  # Remove the remaining centers
  ps.css("center").each do |i|
    i.replace i.inner_html
  end
end

#replace_links(ps, res, lang) ⇒ Object

# File 'lib/bipm/data/importer/common.rb', line 95

def replace_links(ps, res, lang)
  ps.css("a[href]").each do |a|
    href = a.attr("href")

    href = href.gsub(%r'\Ahttps://www\.bipm\.org/', "")

    # Correct links
    href = href.gsub("/web/guest/", "/#{lang}/")

    # Account for some mistakes from an upstream document
    href = href.gsub(%r"\A/jen/", "/en/")
    href = href.gsub(%r"\A/en/CGPM/jsp/", "/en/CGPM/db/")

    href = case href
      when %r'\A/(\w{2})/CGPM/db/(\d+)/(\d+)/(#.*)?\z',
           %r'\A/jsp/(\w{2})/ViewCGPMResolution\.jsp\?CGPM=(\d+)&RES=(\d+)(#.*)?\z',
           %r'\A/(\w{2})/committees/cg/cgpm/(\d+)-\d+/resolution-(\d+)(#.*)?\z',
           "cgpm-resolution:#{$1}/#{$2}/#{$3}#{$4}"
      when %r'\A/(\w{2})/CIPM/db/(\d+)/(\d+)/(#.*)?\z'
        "cipm-resolution:#{$1}/#{$2}/#{$3}#{$4}"
      when %r'\A/(\w{2})/committees/cipm/meeting/([0-9()I]+).html(#.*)?\z'
        "cipm-decisions:#{$1}/#{$2}#{$3}"
      else
        URI(res.uri).merge(href).to_s # Relative -> absolute
      end

    a.set_attribute("href", href)
  end
end