现在的网盘所谓的“极速妙传”，无非就是通过文件校验crc32或md5或者hash值来判断
仔细想来细思极恐，那我的文件和隐私不是被某度看到裤衩子都不剩了？我觉得这样不行（主要是自己爱折腾
于是抽空写了一个自定义的包文件，支持文件压缩/文件加密和大于2G的文件合并成一个文件，并采用.net 8支持windows/linux，理论支持macos
调研 🔗
Stream Encrypt分为对称加密和非对称加密，想来自己的文件也没什么太多的机密信息，使用对称加密即可
那么问题来了，该选什么呢？RC4?SNOW2?ChaCha20-Poly1305?
rc4和snow2之前都有用过，所以觉得采用ChaCha20-Poly1305来实现一下
首先看一下ChaCha20-Poly1305的说明
好了，加密算法选好，压缩该选什么呢？ 7z?zip?gzip?zlib?snappy?
仔细分析一下优劣后，最终选择snappy，然后看一下snappy的说明
分析完毕后具有以后的优势
- 压缩/解压缩速度快
速度快？？好，就这个了，开始
设计 🔗
文件头（40字节）
- 版本号（long） Version
- 文件数量（int32） FileCount
- 打包时间（long） PacketTime
- 12位密钥（byte[12]） Nonce
- 校验值（long） CheckSum
文件实例（长度52+Len字节）
- 文件路径长度（int） Len
- 文件路径（byte[Len]） Path
- 文件大小（long） Size
- 压缩后大小（long） CompressedSize
- 文件偏移（long） Offest
- 创建时间（long） CreationTime
- 编辑时间（long） LastAccessTime
- 最后一次写入时间（long） LastWriteTime
文件数据（CompressedSize）
- 混在一起，通过Offest开始读取，长度以CompressedSize结束
文件头48字节紧跟FileCount * （52 + Len）字节的文件列表
仅加密文件列表，对于文件不加密，小于2G文件压缩，大于2G文件前1024字节加密
如果要做更复杂的接口，可以考虑对文件偏移做手段
此样设计文件大小可以以极低的代价存储 🔗
代码 🔗
这里仅提供核心代码,防止被他人获取以影响目前个人内容
using System.IO.Compression;
using System.Security.Cryptography;
using System.Text;
using chacha20demo.Models;
using Snappier;
namespace chacha20demo
{
public class PacketFile : IDisposable
{
private readonly string _key = "这里填写32位字符串的key";
private readonly int _headerOffest = 0;
private readonly int _entriesOffest = 40;
private readonly object _syncLock = new object();
private Stream _fs;
private BinaryReader _br;
public FileHeader header { get; private set; }
public List<FileEntry> fileEntries { get; set; }
public PacketFile()
{
var random = new Random(Guid.NewGuid().GetHashCode());
header = new FileHeader()
{
PacketTime = DateTime.Now,
Version = 1,
FileCount = 0,
Nonce = GetRandomNonce(),
};
fileEntries = new List<FileEntry>();
}
/// <summary>
/// 获取随机的nonce
/// </summary>
private byte[] GetRandomNonce()
{
using (RandomNumberGenerator rng = RandomNumberGenerator.Create())
{
// 创建一个字节数组来存储随机字节
byte[] randomBytes = new byte[12];
// 使用 RandomNumberGenerator 生成随机字节
rng.GetBytes(randomBytes);
return randomBytes;
}
}
/// <summary>
/// 获取头校验值
/// </summary>
/// <param name="header"></param>
/// <returns></returns>
private long GetHeaderCheckSum(FileHeader header)
{
return header.Version + header.FileCount + header.PacketTime.Ticks + header.Nonce.Sum(x => x);
}
/// <summary>
/// 获取文件体偏移
/// </summary>
/// <param name="fileEntries"></param>
/// <returns></returns>
private long GetFileDataOffest(List<FileEntry> fileEntries)
{
return fileEntries.Sum(x => Encoding.UTF8.GetByteCount(x.Path) + 52) + _entriesOffest;
}
/// <summary>
/// 写入文件头
/// </summary>
private void WriteFileHeader()
{
using (var bw = new BinaryWriter(_fs, Encoding.UTF8, true))
{
bw.Seek(_headerOffest, SeekOrigin.Begin);
bw.Write(header.Version);
bw.Write(header.FileCount);
bw.Write(header.PacketTime.Ticks);
bw.Write(header.Nonce);
bw.Write(header.CheckSum);
}
}
/// <summary>
/// 读取文件头
/// </summary>
private void ReadHeader()
{
_br.BaseStream.Seek(_headerOffest, SeekOrigin.Begin);
var version = _br.ReadInt64();
var fileCount = _br.ReadInt32();
var packetTime = _br.ReadInt64();
var nonce = _br.ReadBytes(12);
var checkSum = _br.ReadInt64();
this.header = new FileHeader() { CheckSum = checkSum, FileCount = fileCount, Nonce = nonce, PacketTime = new DateTime(packetTime), Version = version };
}
/// <summary>
/// 写入文件列表
/// </summary>
private void WriteFileEntrys()
{
ChaCha20 forEncrypting = new ChaCha20(Encoding.UTF8.GetBytes(_key), header.Nonce, (uint)header.FileCount);
using (var psStream = new PacketStream(_fs, forEncrypting))
using (var bw = new BinaryWriter(psStream, Encoding.UTF8))
{
psStream.Seek(_entriesOffest, SeekOrigin.Begin);
foreach (var model in fileEntries)
{
var nameBuffer = Encoding.UTF8.GetBytes(model.Path);
bw.Write(nameBuffer.Length);
bw.Write(nameBuffer);
bw.Write(model.Size);
bw.Write(model.CompressedSize);
bw.Write(model.Offest);
bw.Write(model.CreationTime.Ticks);
bw.Write(model.LastAccessTime.Ticks);
bw.Write(model.LastWriteTime.Ticks);
}
}
}
/// <summary>
/// 读取实例
/// </summary>
private void ReadEntries()
{
fileEntries = new List<FileEntry>();
ChaCha20 forDecrypting = new ChaCha20(Encoding.UTF8.GetBytes(_key), header.Nonce, (uint)header.FileCount);
using (var psStream = new PacketStream(_fs, forDecrypting))
using (var br = new BinaryReader(psStream, Encoding.UTF8, true))
{
psStream.Seek(_entriesOffest, SeekOrigin.Begin);
for (int i = 0; i < this.header.FileCount; i++)
{
var len = br.ReadInt32();
var pathBuffer = br.ReadBytes(len);
var model = new FileEntry();
model.Len = len;
model.Path = Encoding.UTF8.GetString(pathBuffer);
model.Size = br.ReadInt64();
model.CompressedSize = br.ReadInt64();
model.Offest = br.ReadInt64();
model.CreationTime = new DateTime(br.ReadInt64());
model.LastAccessTime = new DateTime(br.ReadInt64());
model.LastWriteTime = new DateTime(br.ReadInt64());
fileEntries.Add(model);
}
}
}
/// <summary>
/// 增加文件
/// </summary>
/// <param name="path"></param>
public void AddFile(string path)
{
lock (_syncLock)
{
FileInfo fi = new FileInfo(path);
this.fileEntries.Add(new FileEntry()
{
Path = path,
LastAccessTime = fi.LastAccessTime,
CreationTime = fi.CreationTime,
LastWriteTime = fi.LastWriteTime,
Size = fi.Length,
});
}
}
/// <summary>
/// 增加目录
/// </summary>
/// <param name="path"></param>
public void AddFloder(string path)
{
var rootPath = Path.GetFullPath(path).Replace("\\", "/").TrimEnd('/') + "/";
foreach (var filePath in Directory.EnumerateFiles(rootPath, "*", SearchOption.AllDirectories))
{
// var relativePath = filePath.Replace(rootPath, "");
this.AddFile(filePath.Replace("\\", "/"));
}
}
/// <summary>
/// 保存文件
/// </summary>
public async Task Save()
{
header.FileCount = fileEntries.Count;
header.CheckSum = GetHeaderCheckSum(this.header);
_fs = new FileStream("./test.lp", FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
long dataOffest = GetFileDataOffest(this.fileEntries);
using (var bw = new BinaryWriter(_fs, Encoding.UTF8, true))
{
bw.Seek((int)dataOffest, SeekOrigin.Begin);
foreach (var item in this.fileEntries)
{
using var fileStream = File.OpenRead(item.Path);
//对于大于2G的文件
if (item.Size >= int.MaxValue)
{
//似乎只需要对头部加密即可，因为常规资源很少有>2G的
//加密文件头前1024个字节即可混淆其文件类型，类似于数据库备份文件,压缩文件,avi等
item.CompressedSize = item.Size;
item.Offest = dataOffest;
//单次读取文件长度kb
long singleReadLenght = 1024 * 1024 * 500;
long fileSize = fileStream.Length;
//流位置
int positon = 1;
var data = new byte[Math.Min(singleReadLenght, fileSize)];
while (positon > 0 && fileSize > 0)
{
positon = await fileStream.ReadAsync(data, 0, data.Length);
//写入二进制数据
bw.Write(data);
//记录写入的二进制流大小
fileSize -= data.Length;
}
continue;
}
using var compressed = new MemoryStream();
using (var compressor = new SnappyStream(compressed, CompressionMode.Compress, false))
{
await fileStream.CopyToAsync(compressor);
await compressor.FlushAsync();
item.CompressedSize = compressed.Length;
item.Offest = dataOffest;
//直接将压缩数据写入文件
compressed.Position = 0;
await compressed.CopyToAsync(_fs);
dataOffest += compressed.Length;
}
}
}
WriteFileEntrys();
WriteFileHeader();
}
/// <summary>
/// 加载包
/// </summary>
/// <param name="path"></param>
public void Load(string path)
{
_fs = new FileStream(path, FileMode.Open, FileAccess.Read);
_br = new BinaryReader(_fs, Encoding.UTF8);
this.ReadHeader();
this.ReadEntries();
}
/// <summary>
/// 导出文件
/// </summary>
/// <param name="fileEntry"></param>
/// <returns></returns>
public async Task Import(FileEntry fileEntry)
{
if (fileEntry.CompressedSize >= int.MaxValue)
{
using (FileStream newfs = File.Create(Path.GetFileName(fileEntry.Path)))
{
int positon = 0;
_fs.Position = fileEntry.Offest;
byte[] buffer = new byte[1024 * 1024 * 500];
while (_fs.Position < fileEntry.Offest + fileEntry.Size)
{
positon = await _fs.ReadAsync(buffer, 0, buffer.Length);
await newfs.WriteAsync(buffer, 0, positon);
}
}
return;
}
else
{
_br.BaseStream.Seek(fileEntry.Offest, SeekOrigin.Begin);
using var compressed = new MemoryStream(_br.ReadBytes((int)fileEntry.CompressedSize));
using var decompressor = new SnappyStream(compressed, CompressionMode.Decompress);
var buffer = new byte[fileEntry.Size];
var bytesRead = decompressor.Read(buffer, 0, buffer.Length);
while (bytesRead > 0)
{
bytesRead = decompressor.Read(buffer, 0, buffer.Length);
}
File.WriteAllBytes(Path.GetFileName(fileEntry.Path), buffer);
}
}
/// <summary>
/// 头加密
/// </summary>
/// <param name="buffer"></param>
public void HeadEncrypt(ref byte[] buffer)
{
}
public void Close()
{
this.Dispose();
}
public void Dispose()
{
lock (_syncLock)
{
try { _br.Close(); }
catch { }
try { _fs.Close(); }
catch { }
}
}
}
}
测试 🔗
Windows下 🔗
Linux(Ubuntu22.04TLS)下 🔗
很好，成功的跑起来了，接下来看看使用Notepad++打开看看是什么效果
至此，打包/解包文件完全按照上述设定进行了加密
可以将一些配置文件资料以此文件打包后上传至某云盘了，再也不怕被偷窥了
本文内所示例的代码并未完全实现，删除了部分自定义加密内容
在这个世界上不存在绝对安全的加密实现，所以为了防止泄露，请确保工具只有您本人持有，这样别人无法反编译/反汇编您的程序即无法解密
结尾 🔗
本文的启蒙为游戏资源包的解包和打包，具体是什么游戏这里不做过多赘述
上述文件结构设计上理论没有什么问题，但是可能有具体的细节没有进行处理