背景概述
客户端加密，是指由用户在本地自行完成文件的加密和解密过程，百度智能云对象存储不参与加密和解密过程，只负责文件的上传、存储、下载过程，明文密钥由用户自行保管在本地。客户端加密增强了文件安全性，即使文件意外泄露，别人也无法解密得到原始数据。但是由于用户需要自行负责保管密钥，如果明文密钥丢失，用户将无法获得原文件内容。本文档提供了一种客户端加密方案。
加密原理
- 本地使用RSA算法生成非对称密钥(private_rsa_key和public_rsa_key)；
- 上传文件时，使用AES 256的CTR模式生成对称密钥（aes_key）；
- 对于普通文件，使用对称密钥aes_key加密原始数据；使用公钥public_rsa_key加密aes_key生成encrypted_aes_key，作为object的meta，并上传文件到Bos。对于大文件，使用Bos的分块上传，每次取一个分块加密并上传，分块大小为16字节的整数倍，同样使用公钥public _rsa_key加密aes_key生成encrypted_aes_key，作为object的meta。
- 下载文件时，首先获取文件的meta信息，得到encrypted_aes_key，利用本地私钥private_rsa_key解密encrypted_aes_key得到aes_key。对于普通文件，下载加密后文件并使用aes_key解密得到原始数据。对于大文件，可以使用分块下载，使用Python SDK的函数get_object(bucket_name,object_key,[range_start,range_end])下载大文件的指定区间[range_start,range_end]中的字节,包含结尾位置range_end处的字节，其中0 ≤range_start≤range_end≤文件大小。
注意：使用AES 256方式加密，AES算法加密时的块大小是128bits=16byte,因此在大文件分块上传时，每个分块大小应该是16字节的整数倍。
加密原理图
Python示例代码
准备工作
1.用户需要安装python SDk，请参考安装python SDK工具包。
2.执行下列命令，安装PyCrypto库。
pip install pycrypto
3.修改配置。修改示例代码中的HOST和AK、SK,作为本示例的配置项。
示例代码如下
#coding=utf-8
'''
The file aims to help client to encrypt data on python sdk with RSA algorithm and symmetric encryption algorithm
'''
import os
import shutil
import base64
import random
import string
#引入配置文件和对象存储模块
from baidubce.bce_client_configuration import BceClientConfiguration
from baidubce.auth.bce_credentials import BceCredentials
from baidubce import exception
from baidubce.services import bos
from baidubce.services.bos import canned_acl
from baidubce.services.bos.bos_client import BosClient
#引入Crypto加密模块
from Crypto import Random
from Crypto.Cipher import AES
from Crypto.Cipher import PKCS1_OAEP
from Crypto.PublicKey import RSA
from Crypto.Util import Counter
#设置对称密钥长度为128bits
_AES256_KEY_SIZE = 32
#设置AES CTR模式的计数器Counter长度
_COUNTER_BITS_LENGTH_AES_CTR = 8*16
class CipherWithAES:
# start为CTR计数器初始值
def __init__(self, key= None, start= None):
if not key:
key = Random.new().read(_AES256_KEY_SIZE)
if not start:
start = random.randint(1,100)
self.key = key
self.start = start
#生成计数器
my_counter = Counter.new(_COUNTER_BITS_LENGTH_AES_CTR, initial_value=self.start)
#生成AES对象
self.cipher = AES.new(self.key, AES.MODE_CTR, counter = my_counter)
#加密数据
def encrypt(self, plaintext):
return self.cipher.encrypt(plaintext)
#解密数据
def decrypt(self, ciphertext):
return self.cipher.decrypt(ciphertext)
class CipherWithRSA:
# 输入参数为公钥文件名和私钥文件名
def __init__(self, public_key_file_name = None, private_key_file_name = None):
self.public_key_file_name=public_key_file_name
self.private_key_file_name = private_key_file_name
if not self.public_key_file_name:
self.public_key_file_name = "rsa_public_key.pem"
if not self.private_key_file_name:
self.private_key_file_name = "rsa_private_key.pem"
#如果没有输入公钥和私钥文件，则本地产生RSA密钥对
if not (os.path.isfile(self.public_key_file_name) and os.path.isfile(self.private_key_file_name)):
self._generate_rsa_key()
return
#从文件读取公钥和私钥，并产生RSA对象
with open(self.public_key_file_name) as file_key:
public_key_obj_rsa = RSA.importKey(file_key.read())
self.encrypt_obj = PKCS1_OAEP.new(public_key_obj_rsa)
with open(self.private_key_file_name) as file_key:
private_key_obj_rsa = RSA.importKey(file_key.read())
self.decrypt_obj = PKCS1_OAEP.new(private_key_obj_rsa)
#本地生成RSA密钥对
def _generate_rsa_key(self):
private_key_obj_rsa = RSA.generate(2048)
public_key_obj_rsa = private_key_obj_rsa.publickey()
self.encrypt_obj = PKCS1_OAEP.new(public_key_obj_rsa)
self.decrypt_obj = PKCS1_OAEP.new(private_key_obj_rsa)
#将生成的密钥对存入本地
with open(self.public_key_file_name,"w") as file_export:
file_export.write(public_key_obj_rsa.exportKey())
with open(self.private_key_file_name,"w") as file_export:
file_export.write(private_key_obj_rsa.exportKey())
#加密数据
def encrypt(self,plaintext_key):
return self.encrypt_obj.encrypt(plaintext_key)
#解密数据
def decrypt(self,ciphertext_key):
return self.decrypt_obj.decrypt(ciphertext_key)
#产生随机文件名
def _random_string(length):
return ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(length))
################put super file#################
#分块上传大文件到百度对象存储，"part_size"为分块大小，必须为16字节的整数倍
def put_super_file(bos_client, bucket_name, super_object_key, super_file, part_size,cipher_rsa):
"""
Put super file to baidu object storage by multipart uploading."part_size" must be multiple of 16 bytes
:param bos_client:bos client
:type bos_client:baidubce.services.bos.bos_client.BosClient
:param bucket_name:None
:type bucket_name:string
:param super_object_key: destion object key of super file
:type super_object_key: string
:param super_file: super file name
:type super_file:string
:param part_size: size of part to upload once,"part_size" must be multiple of 16 bytes and more than 5MB
:type: int
:param cipher_rsa: encrypt symmetric key
:type cipher_rsa: CipherWithRSA
:return :**Http Response**
"""
#1.initial
try:
if not isinstance(bos_client,BosClient):
raise Exception("bos client is None!")
if not (bucket_name and super_object_key):
raise Exception("bucket or object is invalid!")
if not os.path.isfile(super_file):
raise Exception("source file is invalid!")
if not isinstance(cipher_rsa,CipherWithRSA):
raise Exception("cipher_rsa is invalid!")
except Exception as e:
print e
exit()
if not part_size:
part_size = 10*1024*1024
#temp_file加密后的文件分块
temp_file = _random_string(20)
cipher_aes = CipherWithAES()
#2.分块上传
#分块上传分三步，第一步初始化，获取upload_id
upload_id = bos_client.initiate_multipart_upload(
bucket_name = bucket_name,
key = super_object_key,
).upload_id
left_size = os.path.getsize(super_file)
offset = 0
part_number = 1
part_list = []
fs = open(super_file,"r")
while left_size > 0:
if left_size < part_size:
part_size = left_size
# 读取分块并加密
part_content = fs.read(part_size)
encrypted_part_content = cipher_aes.encrypt(part_content)
#分块加密后重新写入temp_file
with open(temp_file,"w") as ft:
ft.write(encrypted_part_content)
#分块上传第二步，上传分块
response = bos_client.upload_part_from_file(
bucket_name, super_object_key, upload_id, part_number, part_size, temp_file, offset)
left_size -= part_size
#保存part number和etag用于调用complete_multipart_upload()
part_list.append({
"partNumber": part_number,
"eTag": response.metadata.etag
})
part_number += 1
os.remove(temp_file)
fs.close()
#使用公钥加密AES密钥，并作为object的meta
user_metadata = {
"key": base64.b64encode(cipher_rsa.encrypt(str(cipher_aes.key))),
"start":base64.b64encode(cipher_rsa.encrypt(str(cipher_aes.start)))
}
#分块上传第三步，完成分块上传
return bos_client.complete_multipart_upload(bucket_name, super_object_key, upload_id,
part_list,user_metadata = user_metadata)
####################### Get super file#######################
#获取大文件的分块，其中range_start, range_end分别为欲获取分块在文件中的起止位置,包含技术位置的字节
def get_part_super_file(bos_client, bucket_name, super_object_key, range_start, range_end,cipher_rsa):
"""
Get part of super file from baidu object storage
:param bos_client:bos client
:type bos_client:baidubce.services.bos.bos_client.BosClient
:param bucket_name:None
:type bucket_name:string
:param super_object_key: destion object key of super file
:type super_object_key: string
:param range_start: index of first bytes of part,minimum value is 0
:type super_file:int
:param range_end: index of last bytes of part
:type: int
:param cipher_rsa: dencrypt symmetric key
:type cipher_rsa: CipherWithRSA
:return :**Http Response**
"""
try:
if not isinstance(bos_client,BosClient):
raise Exception("bos client is None!")
if not (bucket_name and super_object_key):
raise Exception("bucket or object is invalid!")
if not (range_start and range_end):
raise Exception("range is invalid!")
if not isinstance(cipher_rsa,CipherWithRSA):
raise Exception("cipher_rsa is invalid!")
except Exception as e:
print e
exit()
#1.对齐分块起始位置到16字节的整数倍
left_offset = range_start%16
right_offset = 15 -range_end%16
test_range = [range_start-left_offset,range_end+right_offset]
#2.获取object的meta
response = bos_client.get_object_meta_data(bucket_name, super_object_key)
#使用本地公钥来解密AES密钥密文，得到AES密钥明文
download_aes_key = base64.b64decode(getattr(response.metadata,"bce_meta_key"))
download_aes_start = base64.b64decode(str(getattr(response.metadata,"bce_meta_start")))
aes_key = cipher_rsa.decrypt(download_aes_key)
aes_start = cipher_rsa.decrypt(download_aes_start)
#根据欲获取分块的起始位置调整计数器初始值
offset_start = int(aes_start)+range_start/16
cipher_aes = CipherWithAES(aes_key,int(offset_start))
#3. 下载分块密文数据并使用AES密钥解密
response = bos_client.get_object(bucket_name, super_object_key,test_range)
download_content = response.data.read()
plaintext_content = cipher_aes.decrypt(download_content)
#截取用户指定区段的明文数据并返回
return plaintext_content[left_offset:range_end-range_start+left_offset+1]
############# put common file ####################
#加密上传普通文件
def put_common_file(bos_client, bucket_name, object_key, file_name, cipher_rsa):
"""
Put file to baidu object storage
:param bos_client:bos client
:type bos_client:baidubce.services.bos.bos_client.BosClient
:param bucket_name:None
:type bucket_name:string
:param object_key: destion object key of file
:type object_key: string
:param file_name: source file name
:type file_name:string
:param cipher_rsa: encrypt symmetric key
:type cipher_rsa: CipherWithRSA
:return :**Http Response**
"""
try:
if not isinstance(bos_client,BosClient):
raise Exception("bos client is None!")
if not (bucket_name and object_key):
raise Exception("bucket or object is invalid!")
if not os.path.isfile(file_name):
raise Exception("file name is invalid!")
if not isinstance(cipher_rsa,CipherWithRSA):
raise Exception("cipher_rsa is invalid!")
except Exception as e:
print e
exit()
temp_file = _random_string(20)
#读取欲上传文件数据
content=""
with open(file_name,"r") as fp:
content = fp.read()
cipher_aes = CipherWithAES()
#加密数据并写到temp_file
encrypt_content = cipher_aes.encrypt(content)
with open(temp_file,"w") as ft:
ft.write(encrypt_content)
cipher_rsa = CipherWithRSA()
#加密AES密钥作为Object的meta
user_metadata = {
"key": base64.b64encode(cipher_rsa.encrypt(str(cipher_aes.key))),
"start":base64.b64encode(cipher_rsa.encrypt(str(cipher_aes.start)))
}
#上传加密后文件
response = bos_client.put_object_from_file(bucket = bucket_name,
key = object_key,
file_name = temp_file,
user_metadata = user_metadata)
os.remove(temp_file)
return response
################get common file#####################
#下载加密的普通文件
def get_common_file(bos_client, bucket_name, object_key, des_file, cipher_rsa):
"""
Put file to baidu object storage
:param bos_client:bos client
:type bos_client:baidubce.services.bos.bos_client.BosClient
:param bucket_name:None
:type bucket_name:string
:param object_key: destion object key of file
:type object_key: string
:param des_file: destination file name
:type des_file:string
:param cipher_rsa: dencrypt symmetric key
:type cipher_rsa: CipherWithRSA
:return :**Http Response**
"""
try:
if not isinstance(bos_client,BosClient):
raise Exception("bos client is None!")
if not (bucket_name and object_key):
raise Exception("bucket or object is invalid!")
if not des_file:
raise Exception("destination file is invalid!")
if not isinstance(cipher_rsa,CipherWithRSA):
raise Exception("cipher_rsa is invalid!")
except Exception as e:
print e
exit()
#下载获取meta
response = bos_client.get_object_meta_data(bucket_name, object_key)
download_aes_key = base64.b64decode(getattr(response.metadata,"bce_meta_key"))
download_aes_start = base64.b64decode(str(getattr(response.metadata,"bce_meta_start")))
#下载加密数据到本地
download_content = bos_client.get_object_as_string(bucket_name, object_key)
#解密得到AES密钥明文
aes_key = cipher_rsa.decrypt(download_aes_key)
aes_start = cipher_rsa.decrypt(download_aes_start)
cipher_aes = CipherWithAES(aes_key,int(aes_start))
plaintext_content = cipher_aes.decrypt(download_content)
with open(des_file,"w") as fd:
fd.write(plaintext_content)
if __name__ == "__main__":
#以北京地区对象存储为例，替换AK、SK、bucket_name为用户的数据
HOST = 'bj.bcebos.com'
AK = 'Your_Access_Key'
SK = 'Your_Secret_Access_Key'
bucket_name = "Your-Bucket-Name"
super_file= "super_file"
super_object_key = "my-super-object"
#获取bos client
config = BceClientConfiguration(credentials=BceCredentials(AK, SK), endpoint=HOST)
bos_client = BosClient(config)
#1.1上传大文件
#设置分块大小
part_size = 10*1024*1024
cipher_rsa = CipherWithRSA()
#分块上传大文件
put_super_file(bos_client, bucket_name, super_object_key, super_file, part_size,cipher_rsa)
#1.2 获取大文件分块
#设置欲获取的分块起始位置，建议设置成16字节的整数倍
range_start = 16*10
range_end = 16*11-1
#获取明文数据块
result = get_part_super_file(bos_client, bucket_name, super_object_key, range_start, range_end,cipher_rsa)
print "#"*20
print result
print "#"*20
print "length:",len(result)
#2.1上传普通文件
object_key = "myobject"
source_file = "myobject.txt"
put_common_file(bos_client,bucket_name,object_key,source_file,cipher_rsa)
#2.2 下载普通文件
des_file = "des_myobject.txt"
get_common_file(bos_client,bucket_name,object_key,des_file,cipher_rsa)
