In a previous blog post I detailed a simple process for uploading custom images to the Fujitsu K5 IaaS platform. One challenge that I overlooked is large custom image sizes which I’ll address here.
Background: OpenStack Swift objects have a maximum size limitation of 5GB. However the Swift containers can have many thousands of these objects. Files that are larger than 5GB need to be split into a subset of smaller files before being uploaded to the container. When all the component files have been uploaded a zero byte file with a ‘manifest’ header is uploaded to the container. This header is composed of the container name and prefix used to build the component file names. When this zero byte file is referenced through the Swift API all the component files are concatenated and the original large file is downloaded.
The following python script will upload custom images smaller than 1 GB directly into K5’s object storage. Larger files are split into 1GB chunks before uploading to the object storage. The default 1GB size can be changed by using the command line parameters.
Once the image is uploaded it is then registered with the the default K5 project. The image must now be shared with other member projects that wish to consume this image – this will be shared in the next post.
Prerequisites : The script relies on a settings file, k5contractsettings.py, which must contain all your contract details and be placed in the same directory – for example:
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/python | |
| adminUser = 'username' | |
| adminPassword = 'password' | |
| contract = 'contract_name' | |
| contractid = 'contract_id' | |
| defaultid = 'default_project_id' | |
| project = 'working_project' | |
| region = 'uk-1' |
Image Upload Example Script
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/python | |
| # Author : Graham Land | |
| # Date: 08/10/2016 | |
| # | |
| # Purpose: Upload a custom image to K5 Object Storage and then register with K5 Glance | |
| # If the image is greater than 1GB it will be broken down into 1GB chunks | |
| # and then uploaded | |
| # Command line parameters – | |
| # -i image_path | |
| # -c container_name | |
| # -s chunk_size (bytes) | |
| # -n display_name | |
| # -t image_type | |
| # -p project | |
| # | |
| # Prerequisites: k5contractsettings.py file in the same directory with login details | |
| # | |
| # adminUser = 'username' | |
| # adminPassword = 'password' | |
| # contract = 'contract_name' | |
| # contractid = 'contract_id' | |
| # defaultid = 'default_project_id' | |
| # project = 'working_project' | |
| # region = 'uk-1' | |
| # | |
| # blog: https://allthingscloud.eu | |
| # twitter: @allthingsclowd | |
| import sys | |
| import os | |
| import requests | |
| import uuid | |
| import base64 | |
| import time | |
| import getopt | |
| import ntpath | |
| # load your K5 contract details from k5contractsettings.py file | |
| from k5contractsettings import * | |
| # get a scoped auth token | |
| def get_scoped_token(uname,upassword,uproject,udomain,uregion): | |
| identityURL = 'https://identity.' + uregion + '.cloud.global.fujitsu.com/v3/auth/tokens' | |
| response = requests.post(identityURL, | |
| headers={'Content-Type': 'application/json','Accept':'application/json'}, | |
| json={"auth": | |
| {"identity": | |
| {"methods":["password"],"password": | |
| {"user": | |
| {"domain": | |
| {"name":udomain}, | |
| "name":uname, | |
| "password": upassword | |
| }}}, | |
| "scope": | |
| { "project": | |
| {"id":uproject | |
| }}}}) | |
| return response.headers['X-Subject-Token'] | |
| def get_unscoped_token(uname,upassword,udomain,uregion): | |
| identityURL = 'https://identity.' + uregion + '.cloud.global.fujitsu.com/v3/auth/tokens' | |
| response = requests.post(identityURL, | |
| headers={'Content-Type': 'application/json','Accept':'application/json'}, | |
| json={"auth": | |
| {"identity": | |
| {"methods":["password"],"password": | |
| {"user": | |
| {"domain": | |
| {"name":udomain}, | |
| "name":uname, | |
| "password": upassword | |
| }}}}}) | |
| return response.headers['X-Subject-Token'] | |
| # get a central identity portal token | |
| def get_unscoped_idtoken(uname,upassword,udomain): | |
| response = requests.post('https://auth-api.jp-east-1.paas.cloud.global.fujitsu.com/API/paas/auth/token', | |
| headers={'Content-Type': 'application/json'}, | |
| json={"auth": | |
| {"identity": | |
| {"password": | |
| {"user": | |
| {"contract_number":udomain, | |
| "name":uname, | |
| "password": upassword | |
| }}}}}) | |
| return response.headers['X-Access-Token'] | |
| # create a container | |
| def create_new_storage_container(adminUser,adminPassword,project,container_name,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| print scoped_k5token | |
| identityURL = 'https://objectstorage.' + region + '.cloud.global.fujitsu.com/v1/AUTH_' + project + '/' + container_name | |
| print identityURL | |
| response = requests.put(identityURL, | |
| headers={'X-Auth-Token':scoped_k5token,'Content-Type': 'application/json'}) | |
| return response | |
| def upload_file_to_container(adminUser,adminPassword,project,container_name,file_name,file_path,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| uploadfile = open(file_path, 'rb') | |
| data = uploadfile.read() | |
| identityURL = 'https://objectstorage.' + region + '.cloud.global.fujitsu.com/v1/AUTH_' + project + '/' + container_name + '/' + file_name | |
| response = requests.put(identityURL, | |
| data=data, | |
| headers={'X-Auth-Token':scoped_k5token,'Content-Type': 'application/octet-stream'}) | |
| uploadfile.close | |
| return response | |
| def import_from_container_to_k5(adminUser,adminPassword,project,container_name,file_name,display_name,file_path,os_type,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| k5ContainerURL = '/v1/AUTH_' + project + '/' + container_name + '/' + file_name | |
| image_id = str(uuid.uuid4()) | |
| encodedPassword = base64.b64encode(adminPassword) | |
| vmimportURL = 'https://vmimport.' + region + '.cloud.global.fujitsu.com/v1/imageimport' | |
| response = requests.post(vmimportURL, | |
| headers={'X-Auth-Token':scoped_k5token}, | |
| json={"name":display_name, | |
| "location":k5ContainerURL, | |
| "id":image_id, | |
| "conversion": True, | |
| "os_type":os_type, | |
| "user_name":adminUser, | |
| "password":encodedPassword, | |
| "domain_name":contract}) | |
| return response.json() | |
| def verify_image_import_status(adminUser,adminPassword,project,image_id,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| vmimportURL = 'https://vmimport.' + region + '.cloud.global.fujitsu.com/v1/imageimport/' + image_id + '/status' | |
| response = requests.get(vmimportURL, | |
| headers={'X-Auth-Token':scoped_k5token}) | |
| return response.json() | |
| def upload_manifest_to_container(adminUser,adminPassword,project,container_name,file_name,prefix,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| identityURL = 'https://objectstorage.' + region + '.cloud.global.fujitsu.com/v1/AUTH_' + project + '/' + container_name + '/' + file_name | |
| response = requests.put(identityURL, | |
| headers={'X-Auth-Token':scoped_k5token,'X-Object-Manifest': container_name + '/' + prefix}) | |
| return response | |
| # list items in a container | |
| def view_items_in_storage_container(adminUser,adminPassword,project,container_name,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| identityURL = 'https://objectstorage.' + region + '.cloud.global.fujitsu.com/v1/AUTH_' + project + '/' + container_name + '?format=json' | |
| response = requests.get(identityURL, | |
| headers={'X-Auth-Token':scoped_k5token,'Content-Type': 'application/json'}) | |
| return response | |
| # download item in a container | |
| def download_item_in_storage_container(adminUser,adminPassword,project,container_name,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| identityURL = 'https://objectstorage.' + region + '.cloud.global.fujitsu.com/v1/AUTH_' + project + '/' + container_name + '/manifest' | |
| print identityURL | |
| response = requests.get(identityURL, | |
| headers={'X-Auth-Token':scoped_k5token,'Content-Type': 'application/json'}) | |
| return response | |
| def make_out_filename(prefix, idx): | |
| '''Make a filename with a serial number suffix.''' | |
| return prefix + str(idx).zfill(4) | |
| def bsplit(in_filename, bytes_per_file,os_type): | |
| '''Split the input file in_filename into output files of | |
| bytes_per_file bytes each. Last file may have less bytes.''' | |
| in_fil = open(in_filename, "rb") | |
| outfil_idx = 1 | |
| out_filename = make_out_filename(os_type, outfil_idx) | |
| out_fil = open(out_filename, "wb") | |
| byte_count = tot_byte_count = file_count = 0 | |
| c = in_fil.read(1) | |
| # Loop over the input and split it into multiple files | |
| # of bytes_per_file bytes each (except possibly for the | |
| # last file, which may have less bytes. | |
| while c != '': | |
| byte_count += 1 | |
| out_fil.write(c) | |
| # Bump vars; change to next output file. | |
| if byte_count >= bytes_per_file: | |
| tot_byte_count += byte_count | |
| byte_count = 0 | |
| file_count += 1 | |
| out_fil.close() | |
| result = upload_file_to_container(adminUser,adminPassword,defaultid,container_name,out_filename,out_filename,contract,region) | |
| print "Uploaded Package – " + str(file_count) | |
| os.remove(out_filename) | |
| outfil_idx += 1 | |
| out_filename = make_out_filename(os_type, outfil_idx) | |
| out_fil = open(out_filename, "wb") | |
| c = in_fil.read(1) | |
| # Clean up. | |
| in_fil.close() | |
| if not out_fil.closed: | |
| out_fil.close() | |
| result = upload_file_to_container(adminUser,adminPassword,defaultid,container_name,out_filename,out_filename,contract,region) | |
| print "\nUploaded Package – " + str(file_count) | |
| os.remove(out_filename) | |
| if byte_count == 0: | |
| os.remove(out_filename) | |
| # now create manifest file | |
| result = upload_manifest_to_container(adminUser,adminPassword,defaultid,container_name,file_name,os_type,contract,region) | |
| return result | |
| def main(): | |
| try: | |
| # ensure minimium commandline paramaters have been supplied | |
| if (len(sys.argv)<6): | |
| print("Usage1: %s -i 'path_to_image' -c 'container_name' -n 'image_display_name' -p '{project1,project2,project3}' -t [ubuntu|centos|rehhat|win2008SE] [-s 'chunk size in bytes'] " % sys.argv[0]) | |
| sys.exit(2) | |
| # load the command line parameters | |
| myopts, args = getopt.getopt(sys.argv[1:],"i:c:n:p:t:s:",["imagepath=","container=","name=","projects=","type=","size="]) | |
| except getopt.GetoptError: | |
| # if the parameters are incorrect display error message | |
| print("Usage2: %s -i 'path_to_image' -c 'container_name' -n 'image_display_name' -p '{project1,project2,project3}' -t [ubuntu|centos|rehhat|win2008SE] [-s 'chunk size in bytes'] " % sys.argv[0]) | |
| sys.exit(2) | |
| # define global variables from the command line parameters | |
| global container_name | |
| global display_name | |
| global bytes_per_file | |
| global os_type | |
| global file_path | |
| global file_name | |
| # set default chunk size for large images that needs to be broken up must be below 5GB for Swift Object Storage | |
| bytes_per_file = 1048576000 #5242880 #1048576000 #262144000 # 250Mb chunks | |
| ############################### | |
| # o == option | |
| # a == argument passed to the o | |
| ############################### | |
| for o, a in myopts: | |
| if o in ('-i','–imagepath'): | |
| file_path=a | |
| elif o in ('-c','–container'): | |
| container_name=a | |
| elif o in ('-n','–name'): | |
| display_name=a | |
| elif o in ('-p','–projects'): | |
| projects=a | |
| elif o in ('-t','–type'): | |
| os_type=a | |
| elif o in ('-s','–size'): | |
| bytes_per_file=int(a) | |
| else: | |
| print("Usage3: %s -i 'path_to_image' -c 'container_name' -n 'image_display_name' -p '{project1,project2,project3}' -t [ubuntu|centos|rehhat|win2008SE] [-s 'chunk size in bytes'] " % sys.argv[0]) | |
| # extract filename from file path suplied at cli | |
| file_name = ntpath.basename(file_path) | |
| # attempt to read the contents of the container to see if it already exists | |
| result = view_items_in_storage_container(adminUser,adminPassword,defaultid,container_name,contract,region) | |
| # check to see if container already exists, if not then create it | |
| if (result.status_code == 404): | |
| # create container | |
| print "\nCreating new container : " + container_name | |
| result = create_new_storage_container(adminUser,adminPassword,defaultid,container_name,contract,region) | |
| print "\nCreated new container : " + container_name | |
| # check size of file to be uploaded is less than 250GB, if not split into smaller chunks for upload | |
| if (os.path.getsize(file_path) > bytes_per_file): | |
| # loop through image file for multi-part upload | |
| print "\n———- Starting multi-part file upload to K5 object storage —— \n" | |
| result = bsplit(file_path, bytes_per_file,os_type) | |
| print "\n———- Finished multi-part file upload to K5 object storage —— \n" | |
| else: | |
| # simple file upload to container | |
| print "\n———- Starting simple file upload to K5 object storage —— \n" | |
| result = upload_file_to_container(adminUser,adminPassword,defaultid,container_name,file_name,file_path,contract,region) | |
| print "\n———- Finished simple file upload to K5 object storage —— \n" | |
| # list container | |
| print "\n———- List container contents K5 object storage start —— \n" | |
| result = view_items_in_storage_container(adminUser,adminPassword,defaultid,container_name,contract,region) | |
| print result | |
| print "\n———- List container contents K5 object storage end —— \n" | |
| # Register image with K5 | |
| print "\n———- Registering image with K5 —— \n" | |
| result = import_from_container_to_k5(adminUser,adminPassword,defaultid,container_name,file_name,display_name,file_path,os_type,contract,region) | |
| image_id = result['import_id'] | |
| print result | |
| print "\n———- K5 Image import_id : " + image_id + "\n" | |
| # Get import status | |
| print "\n———- Check import status ———- \n\n" | |
| result = verify_image_import_status(adminUser,adminPassword,defaultid,image_id,contract,region) | |
| print result | |
| while ((result['import_status'] != "succeeded") and (result['import_status'] != "failed")): | |
| time.sleep(300) | |
| print "\n———- Check import status ———- \n" | |
| result = verify_image_import_status(adminUser,adminPassword,defaultid,image_id,contract,region) | |
| print result | |
| print "End of Import Process – Import status >>> " + result['import_status'] | |
| if __name__ == "__main__": | |
| main() | |
Example script output:
C:\Users\landg\>python K5ImageUpload.py -i "c:\Users\landg\Downloads\cirrosuploadtest.vmdk" -c uploaddemo12 -n "h ello k5 milti image" -t ubuntu -p NotUsed -s 5242880 Creating new container : uploaddemo12 31c0f27e562c4b3089a546c175c144e4 https://objectstorage.uk-1.cloud.global.fujitsu.com/v1/AUTH_eadb882573ac40b1b101 eac93009a313/uploaddemo12 Created new container : uploaddemo12 ---------- Starting multi-part file upload to K5 object storage ------ Uploaded Package - 1 Uploaded Package - 2 Uploaded Package - 3 Uploaded Package - 4 Uploaded Package - 4 ---------- Finished multi-part file upload to K5 object storage ------ ---------- List container contents K5 object storage start ------ <Response [200]> ---------- List container contents K5 object storage end ------ ---------- Registering image with K5 ------ {u'import_id': u'6a0b58c5-bcda-4a64-919e-23f06b8338ad'} ---------- K5 Image import_id : 6a0b58c5-bcda-4a64-919e-23f06b8338ad ---------- Check import status ---------- {u'conversion': True, u'name': u'hello k5 milti image', u'container_format': u'b are', u'min_ram': u'0', u'ovf_location': u'', u'disk_format': u'raw', u'domain_n ame': u'YssmW1yI', u'location': u'/v1/AUTH_eadb882573ac40b1b101eac93009a313/uplo addemo12/cirrosuploadtest.vmdk', u'min_disk': u'0', u'progress': u'0', u'os_type ': u'ubuntu', u'password': u'*', u'user_name': u'landg', u'id': u'70a38639-f819- 4375-b3d2-cfc99c2a148e', u'import_status': u'queued'} ---------- Check import status ---------- {u'conversion': True, u'name': u'hello k5 milti image', u'container_format': u'b are', u'min_ram': u'0', u'ovf_location': u'', u'disk_format': u'raw', u'domain_n ame': u'YssmW1yI', u'location': u'/v1/AUTH_eadb882573ac40b1b101eac93009a313/uplo addemo12/cirrosuploadtest.vmdk', u'min_disk': u'0', u'progress': 0, u'os_type': u'ubuntu', u'password': u'*', u'user_name': u'landg', u'id': u'70a38639-f819-437 5-b3d2-cfc99c2a148e', u'import_status': u'processing'} ---------- Check import status ---------- {u'container_format': u'bare', u'min_ram': 0, u'updated_at': u'2016-10-08T17:24: 38Z', u'file': u'/v2/images/70a38639-f819-4375-b3d2-cfc99c2a148e/file', u'owner' : u'eadb882573ac40b1b101eac93009a313', u'id': u'70a38639-f819-4375-b3d2-cfc99c2a 148e', u'size': 41126400, u'conversion': True, u'self': u'/v2/images/70a38639-f8 19-4375-b3d2-cfc99c2a148e', u'disk_format': u'raw', u'domain_name': u'YssmW1yI', u'location': u'/v1/AUTH_eadb882573ac40b1b101eac93009a313/uploaddemo12/cirrosupl oadtest.vmdk', u'progress': 100, u'user_name': u'landg', u'schema': u'/v2/schema s/image', u'status': u'active', u'import_status': u'succeeded', u'tags': [], u'v isibility': u'private', u'BaseImageId': u'415b3a0a513aebc27d34c68bd8cdae8c', u'm in_disk': 0, u'password': u'*', u'name': u'hello k5 milti image', u'created_at': u'2016-10-08T17:24:32Z', u'ovf_location': u'', u'fcx.centos': u'true', u'protec ted': False, u'os_type': u'ubuntu'} End of Import Process - Import status >>> succeeded
Happy Stacking!
allthingscloud.eu 