Module: Wp2txt::MemoryMonitor

Defined in:
lib/wp2txt/memory_monitor.rb

Overview

Memory monitoring and adaptive buffer sizing for streaming operations Provides utilities to track memory usage and dynamically adjust buffer sizes

Constant Summary collapse

LOW_MEMORY_THRESHOLD_MB =

Default memory thresholds

256
HIGH_MEMORY_THRESHOLD_MB = 
1024
TARGET_MEMORY_USAGE_PERCENT = 
70
MIN_BUFFER_SIZE =

Buffer size bounds

1_048_576
MAX_BUFFER_SIZE =

1 MB minimum

104_857_600
DEFAULT_BUFFER_SIZE =

100 MB maximum

10_485_760
MEMORY_PER_PROCESS_MB =

Memory required per parallel process (estimated)

300

Class Method Summary collapse

Class Method Details

.available_memory ⇒ Integer

Get available (free) memory in bytes

Returns:

  • (Integer) —

    Available memory in bytes



91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/wp2txt/memory_monitor.rb', line 91

def available_memory
  if File.exist?("/proc/meminfo")
    # Linux: read MemAvailable or estimate from MemFree + Buffers + Cached
    meminfo = File.read("/proc/meminfo")
    if meminfo =~ /^MemAvailable:\s*(\d+)\s*kB/
      return $1.to_i * 1024
    end

    free = buffers = cached = 0
    meminfo.each_line do |line|
      case line
      when /^MemFree:\s*(\d+)\s*kB/
        free = $1.to_i * 1024
      when /^Buffers:\s*(\d+)\s*kB/
        buffers = $1.to_i * 1024
      when /^Cached:\s*(\d+)\s*kB/
        cached = $1.to_i * 1024
      end
    end
    return free + buffers + cached
  else
    # macOS/other: estimate as total - current usage
    total_system_memory - current_memory_usage
  end
end

.current_memory_usage ⇒ Integer

Get current process memory usage in bytes

Returns:

  • (Integer) —

    Memory usage in bytes, or 0 if unavailable



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/wp2txt/memory_monitor.rb', line 23

def current_memory_usage
  if Gem.win_platform?
    # Windows: use tasklist (less reliable)
    begin
      output = IO.popen(["tasklist", "/FI", "PID eq #{Process.pid}", "/FO", "CSV", "/NH"], err: File::NULL, &:read)
      # Parse CSV format: "process.exe","PID","Session","Session#","Mem Usage"
      if output =~ /(\d[\d,]*)\s*K/
        return $1.delete(",").to_i * 1024
      end
    rescue StandardError
      return 0
    end
  else
    # Unix: use /proc or ps
    if File.exist?("/proc/#{Process.pid}/status")
      # Linux: read from /proc
      File.read("/proc/#{Process.pid}/status").each_line do |line|
        if line =~ /^VmRSS:\s*(\d+)\s*kB/
          return $1.to_i * 1024
        end
      end
    else
      # macOS/BSD: use ps
      begin
        output = IO.popen(["ps", "-o", "rss=", "-p", Process.pid.to_s], err: File::NULL, &:read)
        return output.strip.to_i * 1024 unless output.strip.empty?
      rescue StandardError
        return 0
      end
    end
  end
  0
end

.format_memory(bytes) ⇒ String

Format memory size for display

Parameters:

  • bytes (Integer) —

    Size in bytes

Returns:

  • (String) —

    Human-readable size



167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/wp2txt/memory_monitor.rb', line 167

def format_memory(bytes)
  if bytes < 1024
    "#{bytes} B"
  elsif bytes < 1_048_576
    "#{(bytes / 1024.0).round(1)} KB"
  elsif bytes < 1_073_741_824
    "#{(bytes / 1_048_576.0).round(1)} MB"
  else
    "#{(bytes / 1_073_741_824.0).round(2)} GB"
  end
end

.gc_if_needed ⇒ Boolean

Run garbage collection if memory is low

Returns:

  • (Boolean) —

    true if GC was triggered



181
182
183
184
185
186
187
188
 
# File 'lib/wp2txt/memory_monitor.rb', line 181

def gc_if_needed
  if memory_low?
    GC.start
    true
  else
    false
  end
end

.memory_low? ⇒ Boolean

Determine if memory is running low

Returns:

  • (Boolean) —

    true if memory usage is high



128
129
130
131
 
# File 'lib/wp2txt/memory_monitor.rb', line 128

def memory_low?
  available = available_memory / (1024 * 1024) # Convert to MB
  available < LOW_MEMORY_THRESHOLD_MB
end

.memory_stats ⇒ Hash

Get a summary of current memory status

Returns:

  • (Hash) —

    Memory statistics



153
154
155
156
157
158
159
160
161
162
 
# File 'lib/wp2txt/memory_monitor.rb', line 153

def memory_stats
  {
    current_usage_mb: (current_memory_usage / 1_048_576.0).round(2),
    total_system_mb: (total_system_memory / 1_048_576.0).round(2),
    available_mb: (available_memory / 1_048_576.0).round(2),
    usage_percent: memory_usage_percent,
    recommended_buffer_mb: (optimal_buffer_size / 1_048_576.0).round(2),
    low_memory: memory_low?
  }
end

.memory_usage_percent ⇒ Float

Calculate memory usage percentage

Returns:

  • (Float) —

    Percentage of memory used (0-100)



119
120
121
122
123
124
 
# File 'lib/wp2txt/memory_monitor.rb', line 119

def memory_usage_percent
  total = total_system_memory
  return 0.0 if total.zero?

  (current_memory_usage.to_f / total * 100).round(2)
end

.optimal_buffer_size(target_percent: TARGET_MEMORY_USAGE_PERCENT) ⇒ Integer

Calculate optimal buffer size based on available memory

Parameters:

  • target_percent (Integer) (defaults to: TARGET_MEMORY_USAGE_PERCENT) —

    Target memory usage percentage (default: 70%)

Returns:

  • (Integer) —

    Recommended buffer size in bytes



136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/wp2txt/memory_monitor.rb', line 136

def optimal_buffer_size(target_percent: TARGET_MEMORY_USAGE_PERCENT)
  available = available_memory

  # Use a fraction of available memory for buffering
  # Conservative: use only 10% of available memory for buffer
  target_buffer = (available * 0.10).to_i

  # Clamp to reasonable bounds
  target_buffer = MIN_BUFFER_SIZE if target_buffer < MIN_BUFFER_SIZE
  target_buffer = MAX_BUFFER_SIZE if target_buffer > MAX_BUFFER_SIZE

  # Round to nearest MB for cleaner allocation
  ((target_buffer / 1_048_576.0).round * 1_048_576).to_i
end

.optimal_processes(memory_per_process_mb: MEMORY_PER_PROCESS_MB) ⇒ Integer

Calculate optimal number of parallel processes based on CPU and memory

Parameters:

  • memory_per_process_mb (Integer) (defaults to: MEMORY_PER_PROCESS_MB) —

    Estimated memory per process in MB

Returns:

  • (Integer) —

    Recommended number of parallel processes



196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/wp2txt/memory_monitor.rb', line 196

def optimal_processes(memory_per_process_mb: MEMORY_PER_PROCESS_MB)
  cores = Etc.nprocessors

  # CPU-based calculation (scale based on core count)
  cpu_based = case cores
              when 1..4
                [cores - 1, 1].max
              when 5..8
                cores - 2
              else
                # Large systems: use 75% of cores
                (cores * 0.75).to_i
              end

  # Memory-based limit
  available_mb = available_memory / (1024 * 1024)
  memory_based = (available_mb / memory_per_process_mb).to_i

  # Use the smaller of CPU and memory limits, minimum 1
  result = [cpu_based, memory_based].min
  [result, 1].max
end

.parallel_processing_info ⇒ Hash

Get system info for parallel processing decisions

Returns:

  • (Hash) —

    System information



221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/wp2txt/memory_monitor.rb', line 221

def parallel_processing_info
  cores = Etc.nprocessors
  available_mb = (available_memory / 1_048_576.0).round(0)
  optimal = optimal_processes

  {
    cpu_cores: cores,
    available_memory_mb: available_mb,
    memory_per_process_mb: MEMORY_PER_PROCESS_MB,
    optimal_processes: optimal,
    max_by_cpu: cores,
    max_by_memory: (available_mb / MEMORY_PER_PROCESS_MB).to_i
  }
end

.total_system_memory ⇒ Integer

Get total system memory in bytes

Returns:

  • (Integer) —

    Total memory in bytes, or default if unavailable



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/wp2txt/memory_monitor.rb', line 59

def total_system_memory
  if Gem.win_platform?
    # Windows: use wmic
    begin
      output = IO.popen(["wmic", "computersystem", "get", "TotalPhysicalMemory"], err: File::NULL, &:read)
      if output =~ /(\d+)/
        return $1.to_i
      end
    rescue StandardError
      return 4 * 1024 * 1024 * 1024 # Default 4 GB
    end
  elsif File.exist?("/proc/meminfo")
    # Linux
    File.read("/proc/meminfo").each_line do |line|
      if line =~ /^MemTotal:\s*(\d+)\s*kB/
        return $1.to_i * 1024
      end
    end
  else
    # macOS: use sysctl
    begin
      output = IO.popen(["sysctl", "-n", "hw.memsize"], err: File::NULL, &:read)
      return output.strip.to_i unless output.strip.empty?
    rescue StandardError
      return 4 * 1024 * 1024 * 1024 # Default 4 GB
    end
  end
  4 * 1024 * 1024 * 1024 # Default 4 GB
end