【发布时间】：2020-12-22 04:05:39
【问题描述】：
我经常使用一种工具 (Amesim) 将其文件打包到一个未压缩的 tar 文件中。对于版本控制，我通常将文件命名为 file1_Rev01.ame，并随着更改进行迭代。这在我是唯一用户时有效，但最近我更经常地共享文件/模型。尝试共享这些模型很痛苦，它们通常包含非常大的结果（gbs 数据），并且如果难以跟踪版本之间的更改，除非在每次更改时在模型中严格添加文本。 （Amesim 是一个类似于 Simulink 的工具。）
我一直在阅读 git hooks 和 git 过滤器，但我不确定如何更好地管理 tarball 的版本控制。
假设我有文件“my_file.tar”，它由 a.txt、b.model、c.data 和 d.results 组成。
从应用程序端，我将暂存“my_file.tar”并提交提交“模型更新”。在不更改 git 的情况下，它会跟踪对二进制文件的更改。这不可读并且占用大量空间。如果包含结果，则文件非常大。如果持续存储结果，克隆 repo 将具有挑战性。
对于我的第一次尝试，我尝试使用 pre-commit 和 post-checkout 挂钩。
在提交时，我的预提交挂钩将“my_file.tar”解压缩到目录“my_file_tar”中。它会删除来自运行模型的 *.results 文件。无需对此进行跟踪并节省大量空间 (gbs)。
当我拉取模型时，post-checkout 将搜索任何带有 _tar 的文件夹并对其进行 tar，将它们重命名为 my_file.tar。
现在这通常有效。但是，我应该如何处理 my_file.tar 和未压缩的文件夹？如果我在签出后自动删除未压缩的文件夹，git 会指出我有重大更改要跟踪。我每次都需要将文件夹添加/删除到 .gitignore 吗？此外，tar 文件永远不会显示它被跟踪，因为我在预提交代码中删除了它。我能做些什么来清理这个过程？我应该如何以不同的方式处理？
参考资料：
对于此代码，.ame 是一个 tar 文件。
预提交
#!/usr/bin/env python
import argparse
import os
import tarfile
import zipfile
import subprocess
def parse_args():
pass
def log_file(log_item):
cwd = os.getcwd()
file = open("MyFile.txt", "a") # Open file in append mode
file.write(log_item + '\n')
return 1
def get_staged_ame_files():
'''Request a list of staged files from git and return a list of *.ame files
This function opens a subprocess with git, requests a list of names in the git staged list. It will return a list of strings.
'''
out = subprocess.Popen(['git', 'diff', '--staged', '--name-only'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout, stderr = out.communicate()
# Separate output by newlines
# staged_files = stdout.split(b'\n') # split as bytes
# filter for files with .ame
staged_files = stdout.decode('utf-8').split('\n') # split as strings
# Create list of *just* amesim files
staged_ame_files = []
for entry in staged_files:
if entry.endswith(".ame"):
staged_ame_files.append(entry)
if not staged_ame_files:
return None
else:
return staged_ame_files
def extract_ame_files(file_list):
folder_list = []
for list_item in file_list:
# If file exists, extract it. Else continue.
if os.path.isfile(list_item):
tar = tarfile.open(list_item, "r:")
folder_name = list_item[0:-4] + "_ame"
folder_list.append(folder_name)
tar.extractall(path = folder_name)
tar.close()
log_file(folder_name)
else:
print("File {} does not exist.".format(list_item))
return folder_list
def cleanup_ame_ignored_files(folder_list):
'''Removes unecessary files from the folder.
'''
for folder in folder_list:
file_list = os.listdir(folder)
for file in file_list:
if item.endswith(".results"):
os.remove(item)
if item.endswith(".exe"):
os.remove(item)
return 1
def git_add_ame_folders(folders):
# Add *_ame folders to git stage
for folder in folders:
out = subprocess.Popen(['git', 'add', folder + '/'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout, stderr = out.communicate()
# The -u will capture removed files?
out = subprocess.Popen(['git', 'add', '-u', folder + '/'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout, stderr = out.communicate()
log_file(stdout.decode('utf-8'))
return 1
def remove_ame_from_staging(file_list):
# Loop through any staged ame files.
for file in file_list:
out = subprocess.Popen(['git', 'rm', '--cached', file], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout, stderr = out.communicate()
return 1
def main(args=None):
# if file name is *.ame
# extract *.ame as a tar of the same name into a folder of the same name + _ame
# delete .results file
# don't commit .ame file
# Search for files we want to process in the staged list
# These will only be *.ame files.
staged_ame_files = get_staged_ame_files()
if not staged_ame_files:
# If its empty, there's nothing to do. End the function.
return 0
# We're not empty, lets extract each one.
folder_list = extract_ame_files(staged_ame_files)
# Delete all .results files in each extracted folder path
# Stage all files in each folder path
git_add_ame_folders(folder_list)
# Unstage the .ame file
remove_ame_from_staging(staged_ame_files)
return 1
if __name__ == "__main__":
args = parse_args()
main(args)
结帐后
#!/usr/bin/env python
import argparse
import os
import tarfile
import zipfile
import subprocess
import shutil
#from shutil import rmtree # Delete directory trees
def parse_args():
pass
def log_file(log_item):
cwd = os.getcwd()
file = open("MyFile2.txt", "a") # Open file in append mode
file.write(log_item + '\n')
return 1
def compress_ame_files(folder_list):
for list_item in folder_list:
log_file("We're on item {}".format(list_item))
file_name = list_item[0:-4] + ".ame"
log_file("Tar file name {}".format(file_name))
# Delete the file if it exists first.
os.remove(file_name)
with tarfile.open(file_name, "w:") as tar:
tar.add(list_item, arcname=os.path.basename('../'))
return 1
def cleanup_ame_ignored_files(folder_list):
'''Removes unecessary files from the folder.
'''
for folder in folder_list:
file_list = os.listdir(folder)
for file in file_list:
if item.endswith(".results"):
os.remove(item)
if item.endswith(".exe"):
os.remove(item)
return 1
def git_add_ame_folders(folders):
# Add *_ame folders to git stage
for folder in folders:
out = subprocess.Popen(['git', 'add', folder + '/'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout, stderr = out.communicate()
# The -u will capture removed files?
out = subprocess.Popen(['git', 'add', '-u', folder + '/'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout, stderr = out.communicate()
#log_file(stdout.decode('utf-8'))
return 1
def remove_ame_from_staging(file_list):
# Loop through any staged ame files.
for file in file_list:
out = subprocess.Popen(['git', 'rm', '--cached', file], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout, stderr = out.communicate()
return 1
def fast_scandir(dirname):
# https://stackoverflow.com/questions/973473/getting-a-list-of-all-subdirectories-in-the-current-directory?rq=1
subfolders= [f.path for f in os.scandir(dirname) if f.is_dir()]
for dirname in list(subfolders):
subfolders.extend(fast_scandir(dirname))
return subfolders
def delete_ame_folders(folders):
for folder in folders:
try:
shutil.rmtree(folder)
except OSError as e:
print("Error: %s : %s" % (dir_path, e.strerror))
return 1
#def main(args=None):
def main(lines):
print("Post checkout running.")
# find folders with the name _ame
#log_file("We're running.")
folder_list = []
for folder in fast_scandir(os.getcwd()):
if folder.endswith("_ame"):
#log_file("Found folder {}.".format(folder))
folder_list.append(os.path.join(os.getcwd(), folder))
# tar each folder up and rename with .ame
compress_ame_files(folder_list)
# Delete the folders
#delete_ame_folders(folder_list)
return 1
if __name__ == "__main__":
args = parse_args()
main(args)
【问题讨论】：
-
使用 git 过滤器的 Zippey 方法似乎比使用 git hooks 更好。我应该能够使用 tar 来删除 .results 文件而无需提取它......更多内容即将到来。作为额外的奖励，.git 属性可以在 git 目录中进行跟踪，并且更容易与其他队友共享。好像。
标签： git githooks git-filter