Class: ExtremeZip

Inherits:
Object
  • Object
show all
Defined in:
lib/extremezip.zzaqsv.rb

Overview

def checkMemoryUsage

Instance Method Summary collapse

Constructor Details

#initializeExtremeZip

Returns a new instance of ExtremeZip.



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
# File 'lib/extremezip.zzaqsv.rb', line 19

def initialize
    @wholeCbor = {} # 整个CBOR结构
    @vfsDataList = [] # 数据块压缩块列表
    @filePartCounter = 0 # 文件分块计数器
    @responsePipeList = [] # 任务回复管道列表
    @processIdList = [] # 子进程编号列表。

    @maxSubProcessAmount = Etc.nprocessors # 获取最大的子进程个数

    @dataBlockLength = 33554432 # 数据块单元长度, 32MiB

    @clipDownloader = VictoriaFresh.new # 创建下载器。

    @clipDownloader.diskFlush = true # 向磁盘写入缓存
    @clipDownloader.diskMultiFile = true # 写多个磁盘文件
    @clipDownloader.diskFileName = 'victoriafreshdata.v.' # 磁盘文件名前缀
    @clipDownloader.diskFlushSize = @dataBlockLength # 磁盘文件大小
    @clipDownloader.ignoreFileName= '.exzignore' # 设置用于指定忽略文件列表的文件名。
end

Instance Method Details

#addBasicFileInformationObject

加入基本文件信息



50
51
52
53
54
55
# File 'lib/extremezip.zzaqsv.rb', line 50

def addBasicFileInformation
    @wholeCbor['version'] = 233 # 文件格式版本号

    uuid = UUID.new # 获取生成器
    @wholeCbor['uuid'] = uuid.generate # 指定本个压缩包的唯一编号
end

#compressInSubProcess(currentBlockData, currentResponsePipe) ⇒ Object

在子进程中具体执行的压缩代码



160
161
162
163
164
165
166
167
168
169
170
171
172
173
# File 'lib/extremezip.zzaqsv.rb', line 160

def compressInSubProcess(currentBlockData, currentResponsePipe)
    checkMemoryUsage(115) # Debug
    currentBlockDataToCompress = currentBlockData # 读取数据块

    currentCompressedVfsData = LZMA.compress(currentBlockDataToCompress) # 压缩当前块

    checkMemoryUsage(120)
    puts("compressed data length: #{currentCompressedVfsData.bytesize}") # Debug.

    currentResponsePipe.put currentCompressedVfsData # 将压缩后的数据块写入到回复管道中

    checkMemoryUsage(125)
    puts("finished #{Process.pid}") # Debug
end

#compressVfsMenu(victoriaFresh) ⇒ Object

压缩目录数据。



40
41
42
43
44
45
46
47
# File 'lib/extremezip.zzaqsv.rb', line 40

def compressVfsMenu(victoriaFresh)
    replyByteArray = victoriaFresh.to_cbor # #打包成字节数组。

    # 压缩目录数据并放入CBOR:
    compressedVfsMenu = LZMA.compress(replyByteArray) # 压缩目录数据

    @wholeCbor['vfsMenu'] = compressedVfsMenu # 加入目录
end

#exz(rootPath) ⇒ Object

压缩



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
# File 'lib/extremezip.zzaqsv.rb', line 58

def exz(rootPath)
    victoriaFresh, = @clipDownloader.checkOnce(rootPath) # 打包该目录树。

    @filePartAmount = @clipDownloader.currentDiskFlushSuffix # 获取文件个数

    compressVfsMenu(victoriaFresh) # 压缩目录数据。

    addBasicFileInformation # 加入基本文件信息

    processIdList, responsePipeList = launchSubProcesses # 启动子进程。

    receiveCompressedVfsDataList(processIdList, responsePipeList) # 接收压缩后的数据块列表

    checkMemoryUsage(155)
    @wholeCbor['vfsDataList'] = @vfsDataList # 加入数据

    wholeFileContent = 'exz' + "\0" + @wholeCbor.to_cbor # 追加CBOR字节数组

    # 写入文件:
    writeFile(wholeFileContent, victoriaFresh) # 写入文件内容
end

#launchSubProcessesObject

启动子进程。



148
149
150
151
152
153
154
155
156
157
# File 'lib/extremezip.zzaqsv.rb', line 148

def launchSubProcesses

    #while ((@filePartCounter < @filePartAmount) && (true)) # 未处理完毕,并且未达到最大子进程个数
    while ((@filePartCounter < @filePartAmount) && (@filePartCounter<@maxSubProcessAmount)) # 未处理完毕,并且未达到最大子进程个数
        currentResponsePipe, p1 = schedule1Block(@filePartCounter) # 计划一个块的压缩计算

    end # while processDataLength < victoriaFreshData.byte_size do #未处理完毕

    [@processIdList, @responsePipeList]
end

#readBlockFile(filePartCounter) ⇒ Object

读取块文件内容



108
109
110
111
112
113
114
115
116
117
118
# File 'lib/extremezip.zzaqsv.rb', line 108

def readBlockFile(filePartCounter)
    currentBlockFile = File.new(@clipDownloader.diskFileName + filePartCounter.to_s, 'rb') # 打开文件

    currentBlockData = currentBlockFile.read # 读取全部内容

    currentBlockFile.close # 关闭文件

    #File.delete(currentBlockFile) # 删除数据块文件

    currentBlockData
end

#receiveCompressedVfsDataList(processIdList, responsePipeList) ⇒ Object

接收压缩后的数据块列表



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
# File 'lib/extremezip.zzaqsv.rb', line 88

def receiveCompressedVfsDataList(processIdList, responsePipeList)
    processCounter = 0 # 子进程计数器
    
    while (processCounter<@filePartAmount) # 并不是所有分块都被处理完毕了。
        currentSubProcess=processIdList[processCounter] # 获取子进程对象
    #processIdList.each do |currentSubProcess|
        compressed = receiveFromSubProcess(currentSubProcess, responsePipeList, processCounter) # 从子进程中读取数据,并终止子进程

        @vfsDataList << compressed # 加入数据块列表中
        checkMemoryUsage(150)

        processCounter += 1 # 子进程计数
        
        if (@filePartCounter<@filePartAmount) # 还有一些分块尚未交给子进程进行处理
            schedule1Block(@filePartCounter) # 再启动一个子进程
        end # if (@filePartCounter<@filePartAmount) # 还有一些分块尚未交给子进程进行处理
    end # processIdList.each do |currentSubProcess|
end

#receiveFromSubProcess(currentSubProcess, responsePipeList, processCounter) ⇒ Object

从子进程中读取数据,并终止子进程



176
177
178
179
180
181
182
183
184
185
186
187
188
# File 'lib/extremezip.zzaqsv.rb', line 176

def receiveFromSubProcess(currentSubProcess, responsePipeList, processCounter)
    puts("waiting #{currentSubProcess}") # Debug
    checkMemoryUsage(140)

    currentResponsePipe = responsePipeList[processCounter] # 任务回复管道

    currentCompressedVfsDataFromSubProcess = currentResponsePipe.get # 读取压缩后数据
    checkMemoryUsage(145)

    Process.waitpid(currentSubProcess) # 等待该个子进程

    currentCompressedVfsDataFromSubProcess
end

#schedule1Block(filePartCounter) ⇒ Object

计划一个块的压缩计算



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
# File 'lib/extremezip.zzaqsv.rb', line 121

def schedule1Block(filePartCounter)
    currentBlockData = readBlockFile(filePartCounter) # 读取块文件内容

    # currentTaskPipe = Cod.pipe # 任务分配管道
    currentResponsePipe = Cod.pipe # 任务回复管道

    puts("forking sub process, file part counter: #{filePartCounter}") # Debug.

    p1 = fork do # 复制出子进程
        compressInSubProcess(currentBlockData, currentResponsePipe) # 在子进程中具体执行的压缩代码
    end # p1 = fork do #复制出子进程

    # processDataLength += @dataBlockLength # 计数
    checkMemoryUsage(130)

                # 记录管道:
        # taskPipeList << currentTaskPipe
        @responsePipeList << currentResponsePipe # 记录回复管道

        @processIdList << p1 # 记录到子进程列表中

        @filePartCounter += 1 # 计数

    [currentResponsePipe, p1]
end

#writeFile(wholeFileContent, victoriaFresh) ⇒ Object

写入文件内容



81
82
83
84
85
# File 'lib/extremezip.zzaqsv.rb', line 81

def writeFile(wholeFileContent, victoriaFresh)
    extremeZipOutputFile = File.new("#{victoriaFresh['name']}.exz", 'wb') # 创建文件
    extremeZipOutputFile.syswrite(wholeFileContent) # 写入文件
    extremeZipOutputFile.close # 关闭文件
end