From 34c4186e06da7102ae25214c6cc0d72abb1c1516 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 16 Jun 2015 12:23:44 -0700 Subject: [PATCH] Overhaul in terms of readability and definitions are simplified --- api.go | 48 ++++++++++++++++++------------------ lowlevelapi.go | 35 +++++++++++++++------------ lowlevelcommon.go | 32 +++++++++++++++--------- lowleveldefinitions.go | 55 +++++++++++++++++++++++++++--------------- 4 files changed, 99 insertions(+), 71 deletions(-) diff --git a/api.go b/api.go index 53c89f7b4..d29d1022d 100644 --- a/api.go +++ b/api.go @@ -205,7 +205,7 @@ func New(config Config) (API, error) { // Downloads full object with no ranges, if you need ranges use GetPartialObject func (a api) GetObject(bucket, object string) (io.ReadCloser, ObjectStat, error) { // get object - return a.getPartialObject(bucket, object, 0, 0) + return a.getObject(bucket, object, 0, 0) } // GetPartialObject retrieve partial object @@ -215,7 +215,7 @@ func (a api) GetObject(bucket, object string) (io.ReadCloser, ObjectStat, error) // For more information about the HTTP Range header, go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. func (a api) GetPartialObject(bucket, object string, offset, length int64) (io.ReadCloser, ObjectStat, error) { // get partial object - return a.getPartialObject(bucket, object, offset, length) + return a.getObject(bucket, object, offset, length) } // completedParts is a wrapper to make parts sortable by their part number @@ -277,9 +277,9 @@ func (a api) newObjectUpload(bucket, object, contentType string, size int64, dat if err != nil { return err } - completeMultipartUpload.Part = append(completeMultipartUpload.Part, completePart) + completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, completePart) } - sort.Sort(completedParts(completeMultipartUpload.Part)) + sort.Sort(completedParts(completeMultipartUpload.Parts)) _, err = a.completeMultipartUpload(bucket, object, uploadID, completeMultipartUpload) if err != nil { return err @@ -308,7 +308,7 @@ func (a api) listObjectPartsRecursiveInRoutine(bucket, object, uploadID string, } return } - for _, uploadedPart := range listObjectPartsResult.Part { + for _, uploadedPart := range listObjectPartsResult.Parts { ch <- partCh{ Metadata: uploadedPart, Err: nil, @@ -326,7 +326,7 @@ func (a api) listObjectPartsRecursiveInRoutine(bucket, object, uploadID string, } return } - for _, uploadedPart := range listObjectPartsResult.Part { + for _, uploadedPart := range listObjectPartsResult.Parts { ch <- partCh{ Metadata: uploadedPart, Err: nil, @@ -350,7 +350,7 @@ func (a api) continueObjectUpload(bucket, object, uploadID string, size int64, d var completedPart completePart completedPart.PartNumber = part.Metadata.PartNumber completedPart.ETag = part.Metadata.ETag - completeMultipartUpload.Part = append(completeMultipartUpload.Part, completedPart) + completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, completedPart) md5SumBytes, err := hex.DecodeString(strings.Trim(part.Metadata.ETag, "\"")) // trim off the odd double quotes if err != nil { return err @@ -368,9 +368,9 @@ func (a api) continueObjectUpload(bucket, object, uploadID string, size int64, d if err != nil { return err } - completeMultipartUpload.Part = append(completeMultipartUpload.Part, completedPart) + completeMultipartUpload.Parts = append(completeMultipartUpload.Parts, completedPart) } - sort.Sort(completedParts(completeMultipartUpload.Part)) + sort.Sort(completedParts(completeMultipartUpload.Parts)) _, err := a.completeMultipartUpload(bucket, object, uploadID, completeMultipartUpload) if err != nil { return err @@ -379,7 +379,7 @@ func (a api) continueObjectUpload(bucket, object, uploadID string, size int64, d } type multiPartUploadCh struct { - Metadata upload + Metadata multiPartUpload Err error } @@ -394,14 +394,14 @@ func (a api) listMultipartUploadsRecursiveInRoutine(bucket, object string, ch ch listMultipartUploadsResult, err := a.listMultipartUploads(bucket, "", "", object, "", 1000) if err != nil { ch <- multiPartUploadCh{ - Metadata: upload{}, + Metadata: multiPartUpload{}, Err: err, } return } - for _, upload := range listMultipartUploadsResult.Upload { + for _, multiPartUpload := range listMultipartUploadsResult.Uploads { ch <- multiPartUploadCh{ - Metadata: upload, + Metadata: multiPartUpload, Err: nil, } } @@ -413,14 +413,14 @@ func (a api) listMultipartUploadsRecursiveInRoutine(bucket, object string, ch ch listMultipartUploadsResult.NextKeyMarker, listMultipartUploadsResult.NextUploadIDMarker, object, "", 1000) if err != nil { ch <- multiPartUploadCh{ - Metadata: upload{}, + Metadata: multiPartUpload{}, Err: err, } return } - for _, upload := range listMultipartUploadsResult.Upload { + for _, multiPartUpload := range listMultipartUploadsResult.Uploads { ch <- multiPartUploadCh{ - Metadata: upload, + Metadata: multiPartUpload, Err: nil, } } @@ -721,8 +721,8 @@ func (a api) dropIncompleteUploadsInRoutine(bucket, object string, errorCh chan errorCh <- err return } - for _, upload := range listMultipartUploadsResult.Upload { - err := a.abortMultipartUpload(bucket, upload.Key, upload.UploadID) + for _, multiPartUpload := range listMultipartUploadsResult.Uploads { + err := a.abortMultipartUpload(bucket, multiPartUpload.Key, multiPartUpload.UploadID) if err != nil { errorCh <- err return @@ -738,8 +738,8 @@ func (a api) dropIncompleteUploadsInRoutine(bucket, object string, errorCh chan errorCh <- err return } - for _, upload := range listMultipartUploadsResult.Upload { - err := a.abortMultipartUpload(bucket, upload.Key, upload.UploadID) + for _, multiPartUpload := range listMultipartUploadsResult.Uploads { + err := a.abortMultipartUpload(bucket, multiPartUpload.Key, multiPartUpload.UploadID) if err != nil { errorCh <- err return @@ -774,8 +774,8 @@ func (a api) dropAllIncompleteUploadsInRoutine(bucket string, errorCh chan error errorCh <- err return } - for _, upload := range listMultipartUploadsResult.Upload { - err := a.abortMultipartUpload(bucket, upload.Key, upload.UploadID) + for _, multiPartUpload := range listMultipartUploadsResult.Uploads { + err := a.abortMultipartUpload(bucket, multiPartUpload.Key, multiPartUpload.UploadID) if err != nil { errorCh <- err return @@ -791,8 +791,8 @@ func (a api) dropAllIncompleteUploadsInRoutine(bucket string, errorCh chan error errorCh <- err return } - for _, upload := range listMultipartUploadsResult.Upload { - err := a.abortMultipartUpload(bucket, upload.Key, upload.UploadID) + for _, multiPartUpload := range listMultipartUploadsResult.Uploads { + err := a.abortMultipartUpload(bucket, multiPartUpload.Key, multiPartUpload.UploadID) if err != nil { errorCh <- err return diff --git a/lowlevelapi.go b/lowlevelapi.go index 21432df6c..4b4423a5c 100644 --- a/lowlevelapi.go +++ b/lowlevelapi.go @@ -30,11 +30,12 @@ import ( "time" ) +// lowLevelAPI container to hold unexported internal functions type lowLevelAPI struct { config *Config } -// putBucketRequest wrapper creates a new PutBucket request +// putBucketRequest wrapper creates a new putBucket request func (a lowLevelAPI) putBucketRequest(bucket, acl, location string) (*request, error) { var r *request var err error @@ -103,11 +104,13 @@ func (a lowLevelAPI) putBucketRequest(bucket, acl, location string) (*request, e // private - owner gets full access [DEFAULT] // public-read - owner gets full access, others get read access // public-read-write - owner gets full access, others get full access too +// authenticated-read - owner gets full access, authenticated users get read access // ------------------ // // Location valid values // ------------------ // [ us-west-1 | us-west-2 | eu-west-1 | eu-central-1 | ap-southeast-1 | ap-northeast-1 | ap-southeast-2 | sa-east-1 ] +// // Default - US standard func (a lowLevelAPI) putBucket(bucket, acl, location string) error { req, err := a.putBucketRequest(bucket, acl, location) @@ -142,7 +145,7 @@ func (a lowLevelAPI) putBucketACLRequest(bucket, acl string) (*request, error) { return req, nil } -// putBucketACL set the permissions on an existing bucket using access control lists (ACL) +// putBucketACL set the permissions on an existing bucket using Canned ACL's func (a lowLevelAPI) putBucketACL(bucket, acl string) error { req, err := a.putBucketACLRequest(bucket, acl) if err != nil { @@ -246,7 +249,7 @@ func (a lowLevelAPI) getBucketLocation(bucket string) (string, error) { return locationConstraint, nil } -// listObjectsRequest wrapper creates a new ListObjects request +// listObjectsRequest wrapper creates a new listObjects request func (a lowLevelAPI) listObjectsRequest(bucket, marker, prefix, delimiter string, maxkeys int) (*request, error) { // resourceQuery - get resources properly escaped and lined up before using them in http request resourceQuery := func() (*string, error) { @@ -330,6 +333,7 @@ func (a lowLevelAPI) listObjects(bucket, marker, prefix, delimiter string, maxke return listBucketResult, nil } +// headBucketRequest wrapper creates a new headBucket request func (a lowLevelAPI) headBucketRequest(bucket string) (*request, error) { op := &operation{ HTTPServer: a.config.Endpoint, @@ -339,7 +343,7 @@ func (a lowLevelAPI) headBucketRequest(bucket string) (*request, error) { return newRequest(op, a.config, nil) } -// headBucket - useful to determine if a bucket exists and you have permission to access it. +// headBucket useful to determine if a bucket exists and you have permission to access it. func (a lowLevelAPI) headBucket(bucket string) error { if err := invalidBucketToError(bucket); err != nil { return err @@ -379,7 +383,7 @@ func (a lowLevelAPI) headBucket(bucket string) error { return nil } -// deleteBucketRequest wrapper creates a new DeleteBucket request +// deleteBucketRequest wrapper creates a new deleteBucket request func (a lowLevelAPI) deleteBucketRequest(bucket string) (*request, error) { op := &operation{ HTTPServer: a.config.Endpoint, @@ -389,7 +393,8 @@ func (a lowLevelAPI) deleteBucketRequest(bucket string) (*request, error) { return newRequest(op, a.config, nil) } -// deleteBucket - deletes the bucket named in the URI +// deleteBucket deletes the bucket named in the URI +// // NOTE: - // All objects (including all object versions and delete markers) // in the bucket must be deleted before successfully attempting this request @@ -459,8 +464,7 @@ func (a lowLevelAPI) putObjectRequest(bucket, object, contentType string, md5Sum } // putObject - add an object to a bucket -// -// You must have WRITE permissions on a bucket to add an object to it. +// NOTE: You must have WRITE permissions on a bucket to add an object to it. func (a lowLevelAPI) putObject(bucket, object, contentType string, md5SumBytes []byte, size int64, body io.ReadSeeker) (ObjectStat, error) { req, err := a.putObjectRequest(bucket, object, contentType, md5SumBytes, size, body) if err != nil { @@ -481,7 +485,7 @@ func (a lowLevelAPI) putObject(bucket, object, contentType string, md5SumBytes [ return metadata, nil } -// getObjectRequest wrapper creates a new GetObject request +// getObjectRequest wrapper creates a new getObject request func (a lowLevelAPI) getObjectRequest(bucket, object string, offset, length int64) (*request, error) { encodedObject, err := urlEncodeName(object) if err != nil { @@ -507,12 +511,13 @@ func (a lowLevelAPI) getObjectRequest(bucket, object string, offset, length int6 return r, nil } -// getPartialObject - retrieve object from Object Storage +// getObject - retrieve object from Object Storage +// +// Additionally this function also takes range arguments to download the specified +// range bytes of an object. Setting offset and length = 0 will download the full object. // -// Additionally it also takes range arguments to download the specified range bytes of an object. -// Setting offset and length = 0 will download the full object. // For more information about the HTTP Range header, go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. -func (a lowLevelAPI) getPartialObject(bucket, object string, offset, length int64) (io.ReadCloser, ObjectStat, error) { +func (a lowLevelAPI) getObject(bucket, object string, offset, length int64) (io.ReadCloser, ObjectStat, error) { if err := invalidArgumentToError(object); err != nil { return nil, ObjectStat{}, err } @@ -620,7 +625,7 @@ func (a lowLevelAPI) headObjectRequest(bucket, object string) (*request, error) return newRequest(op, a.config, nil) } -// headObject - retrieves metadata from an object without returning the object itself +// headObject retrieves metadata from an object without returning the object itself func (a lowLevelAPI) headObject(bucket, object string) (ObjectStat, error) { if err := invalidBucketToError(bucket); err != nil { return ObjectStat{}, err @@ -681,7 +686,7 @@ func (a lowLevelAPI) headObject(bucket, object string) (ObjectStat, error) { /// Service Operations -// listBucketRequest wrapper creates a new ListBuckets request +// listBucketRequest wrapper creates a new listBuckets request func (a lowLevelAPI) listBucketsRequest() (*request, error) { op := &operation{ HTTPServer: a.config.Endpoint, diff --git a/lowlevelcommon.go b/lowlevelcommon.go index 3f708fe0a..845348199 100644 --- a/lowlevelcommon.go +++ b/lowlevelcommon.go @@ -30,10 +30,12 @@ import ( "unicode/utf8" ) +// decoder provides a unified decoding method interface type decoder interface { Decode(v interface{}) error } +// acceptTypeDecoder provide decoded value in given acceptType func acceptTypeDecoder(body io.Reader, acceptType string, v interface{}) error { var decoder decoder switch { @@ -47,22 +49,28 @@ func acceptTypeDecoder(body io.Reader, acceptType string, v interface{}) error { return decoder.Decode(v) } -// urlEncodedName- encode the strings from UTF-8 byte representations to HTML hex escape sequences -func urlEncodeName(objectName string) (string, error) { +// urlEncodedName encode the strings from UTF-8 byte representations to HTML hex escape sequences +// +// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 +// non english characters cannot be parsed due to the nature in which url.Encode() is written +// +// This function on the other hand is a direct replacement for url.Encode() technique to support +// pretty much every UTF-8 character. +func urlEncodeName(name string) (string, error) { // if object matches reserved string, no need to encode them reservedNames := regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") - if reservedNames.MatchString(objectName) { - return objectName, nil + if reservedNames.MatchString(name) { + return name, nil } - var encodedObjectName string - for _, s := range objectName { + var encodedName string + for _, s := range name { if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) - encodedObjectName = encodedObjectName + string(s) + encodedName = encodedName + string(s) continue } switch s { case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) - encodedObjectName = encodedObjectName + string(s) + encodedName = encodedName + string(s) continue default: len := utf8.RuneLen(s) @@ -73,14 +81,14 @@ func urlEncodeName(objectName string) (string, error) { utf8.EncodeRune(u, s) for _, r := range u { hex := hex.EncodeToString([]byte{r}) - encodedObjectName = encodedObjectName + "%" + strings.ToUpper(hex) + encodedName = encodedName + "%" + strings.ToUpper(hex) } } } - return encodedObjectName, nil + return encodedName, nil } -// sum256Reader calculate sha256 sum for an input reader +// sum256Reader calculate sha256 sum for an input read seeker func sum256Reader(reader io.ReadSeeker) ([]byte, error) { h := sha256.New() var err error @@ -117,7 +125,7 @@ func sumHMAC(key []byte, data []byte) []byte { return hash.Sum(nil) } -// sumMD5Reader calculate md5 for an input reader of a given size +// sumMD5Reader calculate md5 for an input read seeker of a given size func sumMD5Reader(body io.ReadSeeker, size int64) ([]byte, error) { hasher := md5.New() _, err := io.CopyN(hasher, body, size) diff --git a/lowleveldefinitions.go b/lowleveldefinitions.go index deb2fb0db..1b86d1394 100644 --- a/lowleveldefinitions.go +++ b/lowleveldefinitions.go @@ -21,7 +21,7 @@ import ( "time" ) -// listAllMyBucketsResult container for ListBucets response +// listAllMyBucketsResult container for listBuckets response type listAllMyBucketsResult struct { // Container for one or more buckets. Buckets struct { @@ -36,14 +36,14 @@ type owner struct { ID string } -// commonPrefix container for prefix response in ListObjects +// commonPrefix container for prefix response type commonPrefix struct { Prefix string } -// listBucketResult container for ListObjects response +// listBucketResult container for listObjects response type listBucketResult struct { - CommonPrefixes []commonPrefix // A response can contain CommonPrefixes only if you specify a delimiter + CommonPrefixes []commonPrefix // A response can contain CommonPrefixes only if you have specified a delimiter Contents []ObjectStat // Metadata about each object returned Delimiter string @@ -68,15 +68,21 @@ type listBucketResult struct { Prefix string } -type upload struct { - Key string - UploadID string `xml:"UploadId"` +// multiPartUpload container for multipart session +type multiPartUpload struct { + // Date and time at which the multipart upload was initiated. + Initiated time.Time `type:"timestamp" timestampFormat:"iso8601"` Initiator initiator Owner owner StorageClass string - Initiated time.Time `type:"timestamp" timestampFormat:"iso8601"` + + // Key of the object for which the multipart upload was initiated. + Key string + + // Upload ID that identifies the multipart upload. + UploadID string `xml:"UploadId"` } // listMultipartUploadsResult container for ListMultipartUploads response @@ -89,7 +95,7 @@ type listMultipartUploadsResult struct { EncodingType string MaxUploads int64 IsTruncated bool - Upload []upload + Uploads []multiPartUpload `xml:"Upload"` Prefix string Delimiter string CommonPrefixes []commonPrefix // A response can contain CommonPrefixes only if you specify a delimiter @@ -103,13 +109,20 @@ type initiator struct { // partMetadata container for particular part of an object type partMetadata struct { - PartNumber int + // Part number identifies the part. + PartNumber int + + // Date and time the part was uploaded. LastModified time.Time - ETag string - Size int64 + + // Entity tag returned when the part was uploaded, usually md5sum of the part + ETag string + + // Size of the uploaded part data. + Size int64 } -// listObjectPartsResult container for ListObjectParts response +// listObjectPartsResult container for ListObjectParts response. type listObjectPartsResult struct { Bucket string Key string @@ -125,19 +138,19 @@ type listObjectPartsResult struct { // Indicates whether the returned list of parts is truncated. IsTruncated bool - Part []partMetadata + Parts []partMetadata `xml:"Part"` EncodingType string } -// initiateMultipartUploadResult container for InitiateMultiPartUpload response +// initiateMultipartUploadResult container for InitiateMultiPartUpload response. type initiateMultipartUploadResult struct { Bucket string Key string UploadID string `xml:"UploadId"` } -// completeMultipartUploadResult container for completed multipart upload response +// completeMultipartUploadResult container for completed multipart upload response. type completeMultipartUploadResult struct { Location string Bucket string @@ -145,17 +158,19 @@ type completeMultipartUploadResult struct { ETag string } -// completePart sub container lists individual part numbers and their md5sum, part of CompleteMultipartUpload +// completePart sub container lists individual part numbers and their md5sum, part of completeMultipartUpload. type completePart struct { - XMLName xml.Name `xml:"http://doc.s3.amazonaws.com/2006-03-01 Part" json:"-"` + XMLName xml.Name `xml:"http://doc.s3.amazonaws.com/2006-03-01 Part" json:"-"` + + // Part number identifies the part. PartNumber int ETag string } // completeMultipartUpload container for completing multipart upload type completeMultipartUpload struct { - XMLName xml.Name `xml:"http://doc.s3.amazonaws.com/2006-03-01 CompleteMultipartUpload" json:"-"` - Part []completePart + XMLName xml.Name `xml:"http://doc.s3.amazonaws.com/2006-03-01 CompleteMultipartUpload" json:"-"` + Parts []completePart `xml:"Part"` } // createBucketConfiguration container for bucket configuration