Module: PythonWorkflow

Defined in:
lib/scout/workflow/python.rb,
lib/scout/workflow/python/task.rb,
lib/scout/workflow/python/inputs.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#python_task_dir ⇒ Object

Returns the value of attribute python_task_dir.



9
10
11
 
# File 'lib/scout/workflow/python.rb', line 9

def python_task_dir
  @python_task_dir
end

Class Method Details

.build_python_argv(py_params, values) ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/scout/workflow/python/inputs.rb', line 2

def self.build_python_argv(py_params, values)
  argv = []
  py_params.each do |p|
    name = p['name']
    ptype = p['type']
    default = p['default']
    required = p['required']
    val = values[name]

    if val.nil?
      next unless required
      next
    end

    flag = "--#{name}"
    if ptype.start_with?('list[')
      # Accept several input formats for lists:
      # - Ruby Array
      # - Comma-separated string: "a,b,c"
      # - Path to a file: "file.txt" -> read lines
      items = []
      if val.is_a?(String)
        # If file exists, read lines
        if File.exist?(val)
          items = File.readlines(val, chomp: true)
        elsif val.include?(',')
          items = val.split(',').map(&:strip)
        else
          items = [val]
        end
      elsif val.respond_to?(:to_ary)
        items = Array(val).map(&:to_s)
      else
        items = [val.to_s]
      end

      # pass flag once followed by all items (argparse with nargs='+' expects this)
      argv << flag
      items.each do |x|
        argv << x.to_s
      end
    elsif ptype == 'boolean'
      if default == true
        argv << "--no-#{name}" unless val
      else
        argv << flag if val
      end
    else
      # For scalar inputs: if given a file path and the file exists, pass the path
      # as-is (the Python side can decide how to handle it). Keep quoting to
      # preserve spaces when later shell-joining.
      argv << flag
      argv << val.to_s
    end
  end
  argv
end

.load_directory(path = nil, workflow_name = nil) ⇒ Object 



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/scout/workflow/python.rb', line 15

def self.load_directory(path = nil, workflow_name = nil)
  workflow = begin
                 m = Module.new
                 m.extend Workflow
                 m.extend PythonWorkflow
                 m.name = workflow_name || "PythonWorkflow"
                 m.tasks = {}
                 m
               end

  Kernel.const_set workflow_name, workflow

  path = Scout.python.task if path.nil?

  workflow.python_task_dir = path
  path.glob_names("*.py").each do |name|
    name = name.sub '.py', ''
    workflow.python_task name
  end

  workflow
end

.map_param(p) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/scout/workflow/python/task.rb', line 30

def self.map_param(p)
  desc = p['help'] || ""
  default = p['default']
  required = p['required'] ? true : false

  ruby_type =
    case p['type']
    when 'string' then :string
    when 'integer' then :integer
    when 'float'   then :float
    when 'boolean' then :boolean
    when 'binary'  then :binary
    when 'path'    then :file
    else
      if p['type'].start_with?('list[')
        subtype = p['type'][5..-2]
        ruby_sub = case subtype
                   when 'path' then :file_array
                   else :array
                   end
        ruby_sub
      else
        :string
      end
    end

  options = {}
  options[:required] = true if required

  { name: p['name'], type: ruby_type, desc: desc, default: default, options: options }
end

.map_returns(py_type) ⇒ Object 



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/scout/workflow/python/task.rb', line 11

def self.map_returns(py_type)
  case py_type
  when nil then :string
  when 'string' then :string
  when 'integer' then :integer
  when 'float' then :float
  when 'boolean' then :boolean
  when 'binary' then :binary
  when 'path' then :string
  when 'list', 'array' then :array
  else
    if py_type.start_with?("list[")
      :array
    else
      :string
    end
  end
end

.read_python_metadata(file) ⇒ Object 



5
6
7
8
9
 
# File 'lib/scout/workflow/python/task.rb', line 5

def self.read_python_metadata(file)
  out = ScoutPython.run_file file, '--scout-metadata'
  raise "Error getting metadata from #{file}: #{err}" unless out.exit_status == 0
  JSON.parse(out.read)
end

Instance Method Details

#python_task(task_sym, file: nil, returns: nil, extension: nil, desc: nil) ⇒ Object 



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/scout/workflow/python/task.rb', line 62

def python_task(task_sym, file: nil, returns: nil, extension: nil, desc: nil)
  name = task_sym.to_s
  file ||= python_task_dir[name].find_with_extension('py')
  raise "Python task file not found: #{file}" unless File.exist?(file)

  metas = PythonWorkflow.read_python_metadata(file)
  metas = [metas] unless Array === metas

  # For each function defined in the python file, register a workflow task
  metas.each do |meta|
    meta['returns'] = returns.to_s if returns
    task_desc = desc || meta['description']

    ruby_returns = PythonWorkflow.map_returns(meta['returns'])
    if meta['params'].nil?
      ruby_inputs  = []
    else
      ruby_inputs  = meta['params'].map { |p| PythonWorkflow.map_param(p) }
    end

    ruby_inputs.each do |inp|
      input(inp[:name].to_sym, inp[:type], inp[:desc], inp[:default], inp[:options] || {})
    end

    self.desc(task_desc) if task_desc && !task_desc.empty?
    self.extension(extension) if extension

    task({ meta['name'].to_sym => ruby_returns }) do |*args|
      arg_names = ruby_inputs.map { |i| i[:name] }
      values = {}
      arg_names.each_with_index { |n,i| values[n] = args[i] }

      argv = PythonWorkflow.build_python_argv(meta['params'], values)
      # prefix with function name so the python script runs the desired function
      full_argv = [meta['name']] + argv

      out = ScoutPython.run_file file, Shellwords.shelljoin(full_argv)
      # out is expected to respond to exit_status and read
      raise "Python task #{meta['name']} failed" unless out.exit_status == 0
      txt = out.read.to_s
      # try JSON
      begin
        next JSON.parse(txt)
      rescue JSON::ParserError
        # not JSON; for list returns, split by newline
        if ruby_returns == :array || ruby_returns == :file_array
          next txt.split("\n").map(&:to_s)
        end
        next txt.strip
      end
    end
  end
end