From 4bf9dced7508742ba875034819e13d998aa89539 Mon Sep 17 00:00:00 2001 From: Jamie Cressey Date: Mon, 27 Jun 2016 13:11:07 +0100 Subject: [PATCH] Changes for private S3 packaging --- LICENSE | 2 +- README.md | 36 +-- cloudformation/s3-iam-bucket-policy.json | 29 -- cloudformation/s3-pypi-with-waf.json | 305 --------------------- cloudformation/s3-pypi.json | 246 ----------------- pypis3/__init__.py | 2 + {s3pypi => pypis3}/cli.py | 13 +- {s3pypi => pypis3}/package.py | 6 +- {s3pypi => pypis3}/storage.py | 9 +- {s3pypi => pypis3}/templates/index.html.j2 | 0 s3pypi/__init__.py | 2 - setup.py | 12 +- tests/unit/test_storage.py | 6 - 13 files changed, 35 insertions(+), 633 deletions(-) delete mode 100644 cloudformation/s3-iam-bucket-policy.json delete mode 100644 cloudformation/s3-pypi-with-waf.json delete mode 100644 cloudformation/s3-pypi.json create mode 100644 pypis3/__init__.py rename {s3pypi => pypis3}/cli.py (69%) rename {s3pypi => pypis3}/package.py (95%) rename {s3pypi => pypis3}/storage.py (86%) rename {s3pypi => pypis3}/templates/index.html.j2 (100%) delete mode 100644 s3pypi/__init__.py delete mode 100644 tests/unit/test_storage.py diff --git a/LICENSE b/LICENSE index abd14ba..5d13dc2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2016 November Five +Copyright (c) 2016 Jamie Cressey Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 603585a..69ee61e 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,7 @@ -S3PyPI +PyPiS3 ====== -S3PyPI is a CLI tool for creating a Python Package Repository in an S3 bucket. - -An extended tutorial on using this tool can be found [here](https://novemberfive.co/blog/opensource-pypi-package-repository-tutorial/). +PyPiS3 is a fork of [S3PyPI](https://github.com/novemberfiveco/s3pypi) is a CLI tool for creating a Python Package Repository in an S3 bucket, adapted for use as a private PyPi repository entirely backed by S3 storage. Installation @@ -12,54 +10,46 @@ Installation Install the latest version: ```bash -pip install --upgrade s3pypi +pip install --upgrade pypis3 ``` Install the development version: ```bash -git clone -b develop git@github.com:novemberfiveco/s3pypi.git -cd s3pypi/ && sudo pip install -e . +git clone -b develop git@github.com:jamiecressey/pypis3.git +cd pypis3/ && sudo pip install -e . ``` -Setting up S3 and CloudFront +Setting up S3 and S3Auth ---------------------------- -Before you can start using ``s3pypi``, you must set up an S3 bucket for your Python Package Repository, with static website hosting enabled. Additionally, you need a CloudFront distribution for serving the packages in your S3 bucket to ``pip`` over HTTPS. Both of these resources can be created using the CloudFormation templates provided in the ``cloudformation/`` directory: - -```bash -aws cloudformation create-stack --stack-name STACK_NAME \ - --template-body file://cloudformation/s3-pypi.json \ - --parameters ParameterKey=ServerCertificateId,ParameterValue=SERVER_CERT_ID \ - ParameterKey=DomainName,ParameterValue=DOMAIN_NAME -``` - -[Managing Your Server Certificates](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_server-certs_manage.html) +Before you can start using ``pypis3``, you must set up a S3Auth enabled bucket. Following the steps on their website: http://www.s3auth.com/ Distributing packages --------------------- -You can now use ``s3pypi`` to create Python packages and upload them to your S3 bucket. To hide packages from the public, you can specify a secret subdirectory using the ``--secret`` option: +You can now use ``pypis3`` to create Python packages and upload them to your S3 bucket: ```bash cd /path/to/your/awesome-project/ -s3pypi --bucket mybucket [--secret SECRET] +pypis3 --bucket mybucket --url pypi.example.com ``` Installing packages ------------------- -Install your packages using ``pip`` by pointing the ``--extra-index-url`` to your CloudFront distribution (optionally followed by a secret subdirectory): +Install your packages using ``pip`` by pointing the ``--extra-index-url`` to your S3Auth enabled bucket: ```bash -pip install --upgrade awesome-project --extra-index-url https://pypi.example.com/SECRET/ +pip install --upgrade awesome-project --extra-index-url http://pypi.example.com/ ``` Alternatively, you can configure the index URL in ``~/.pip/pip.conf``: ``` [global] -extra-index-url = https://pypi.example.com/SECRET/ +extra-index-url = http://pypi.example.com ``` + diff --git a/cloudformation/s3-iam-bucket-policy.json b/cloudformation/s3-iam-bucket-policy.json deleted file mode 100644 index 8b4a69d..0000000 --- a/cloudformation/s3-iam-bucket-policy.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowUserToSeeBucketListInTheConsole", - "Action": [ - "s3:ListAllMyBuckets", - "s3:GetBucketLocation" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:s3:::*" - ] - }, - { - "Sid": "AllowPutActionInBucket", - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:PutObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:aws:s3:::pypi.example.com/", - "arn:aws:s3:::pypi.example.com/*" - ] - } - ] -} \ No newline at end of file diff --git a/cloudformation/s3-pypi-with-waf.json b/cloudformation/s3-pypi-with-waf.json deleted file mode 100644 index 8244bc5..0000000 --- a/cloudformation/s3-pypi-with-waf.json +++ /dev/null @@ -1,305 +0,0 @@ -{ - "AWSTemplateFormatVersion": "2010-09-09", - "Description": "s3pypi setup - Visit https://novemberfive.co.", - "Mappings": { - "RegionMap": { - "us-east-1": { - "WebsiteEndpoint": "s3-website-us-east-1.amazonaws.com" - }, - "us-west-2": { - "WebsiteEndpoint": "s3-website-us-west-2.amazonaws.com" - }, - "us-west-1": { - "WebsiteEndpoint": "s3-website-us-west-1.amazonaws.com" - }, - "eu-west-1": { - "WebsiteEndpoint": "s3-website-eu-west-1.amazonaws.com" - }, - "eu-central-1": { - "WebsiteEndpoint": "s3-website.eu-central-1.amazonaws.com" - }, - "ap-southeast-1": { - "WebsiteEndpoint": "s3-website-ap-southeast-1.amazonaws.com" - }, - "ap-northeast-1": { - "WebsiteEndpoint": "s3-website-ap-northeast-1.amazonaws.com" - }, - "ap-southeast-2": { - "WebsiteEndpoint": "s3-website-ap-southeast-2.amazonaws.com" - }, - "ap-northeast-2": { - "WebsiteEndpoint": "s3-website.ap-northeast-2.amazonaws.com" - }, - "sa-east-1": { - "WebsiteEndpoint": "s3-website-sa-east-1.amazonaws.com" - } - } - }, - "Outputs": { - "CNAMERecordValue": { - "Description": "The value of the CNAME or alias record of the configured (sub)domain.", - "Value": { - "Fn::GetAtt": [ - "PyPiCloudfrontDistribution", - "DomainName" - ] - } - } - }, - "Parameters": { - "DomainName": { - "Description": "The (sub)domain that you want to use for your PyPi repository (e.g. pypi.yourcompany.com)", - "Type": "String" - }, - "ServerCertificateId": { - "Description": "ID of the server certificate for the domain name (e.g. ASCAXXXXXXXXXXXXXXXXX)", - "Type": "String" - }, - "WhitelistedCIDRBlock": { - "Description": "Whitelisted IP or IP range (Format: IPv4, CIDR notation, e.g. 10.0.0.1/32)", - "Type": "String", - "AllowedPattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(/([0-9]|[1-2][0-9]|3[0-2]))$" - } - }, - "Resources": { - "PyPiS3Bucket": { - "Type": "AWS::S3::Bucket", - "Properties": { - "BucketName": { - "Ref": "DomainName" - }, - "WebsiteConfiguration": { - "IndexDocument": "index.html" - } - } - }, - "PyPiS3BucketBucketPolicy": { - "Type": "AWS::S3::BucketPolicy", - "Properties": { - "Bucket": { - "Ref": "PyPiS3Bucket" - }, - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetObject" - ], - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:aws:s3:::", - { - "Ref": "PyPiS3Bucket" - }, - "/*" - ] - ] - }, - "Principal": "*", - "Condition": { - "IpAddress": { - "aws:SourceIp": [ - "52.84.0.0/15", - "54.182.0.0/16", - "54.192.0.0/16", - "54.230.0.0/16", - "54.239.128.0/18", - "54.239.192.0/19", - "54.240.128.0/18", - "204.246.164.0/22", - "204.246.168.0/22", - "204.246.174.0/23", - "204.246.176.0/20", - "205.251.254.0/24", - "205.251.252.0/23", - "205.251.250.0/23", - "205.251.249.0/24", - "205.251.192.0/19", - "216.137.32.0/19" - ] - } - } - } - ] - } - } - }, - "PyPiCloudfrontDistribution": { - "Type": "AWS::CloudFront::Distribution", - "Properties": { - "DistributionConfig": { - "WebACLId": { - "Ref": "PyPiWebACL" - }, - "Aliases": [ - { - "Ref": "DomainName" - } - ], - "Comment": "Created by s3pypi", - "DefaultCacheBehavior": { - "AllowedMethods": [ - "GET", - "HEAD" - ], - "ForwardedValues": { - "QueryString": "true" - }, - "TargetOriginId": "PyPiS3BucketOrigin", - "ViewerProtocolPolicy": "https-only" - }, - "Enabled": "true", - "Origins": [ - { - "CustomOriginConfig": { - "HTTPPort": "80", - "HTTPSPort": "443", - "OriginProtocolPolicy": "http-only" - }, - "DomainName": { - "Fn::Join": [ - ".", - [ - { - "Ref": "PyPiS3Bucket" - }, - { - "Fn::FindInMap": [ - "RegionMap", - { - "Ref": "AWS::Region" - }, - "WebsiteEndpoint" - ] - } - ] - ] - }, - "Id": "PyPiS3BucketOrigin" - } - ], - "PriceClass": "PriceClass_100", - "ViewerCertificate": { - "IamCertificateId": { - "Ref": "ServerCertificateId" - }, - "MinimumProtocolVersion": "TLSv1", - "SslSupportMethod": "sni-only" - } - } - } - }, - "PyPiIPWhitelist": { - "Type": "AWS::WAF::IPSet", - "Properties": { - "Name": "PyPi Whitelisted IPs", - "IPSetDescriptors": [ - { - "Type": "IPV4", - "Value": { - "Ref": "WhitelistedCIDRBlock" - } - } - ] - } - }, - "PyPiWAFAllowedIPRule": { - "Type": "AWS::WAF::Rule", - "Properties": { - "MetricName": "PyPiWAFAllowedIPRule", - "Name": "PyPiWAFAllowedIPRule", - "Predicates": [ - { - "DataId": { - "Ref": "PyPiIPWhitelist" - }, - "Negated": false, - "Type": "IPMatch" - } - ] - } - }, - "PyPiWebACL": { - "Type": "AWS::WAF::WebACL", - "Properties": { - "Name": "Created by s3pypi", - "DefaultAction": { - "Type": "BLOCK" - }, - "MetricName": "PyPiWebACL", - "Rules": [ - { - "Action": { - "Type": "ALLOW" - }, - "Priority": 1, - "RuleId": { - "Ref": "PyPiWAFAllowedIPRule" - } - } - ] - } - }, - "PublishS3PyPiPackages": { - "Type": "AWS::IAM::ManagedPolicy", - "Properties": { - "Description": "Policy for updating packages in S3", - "Path": "/", - "PolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowUserToSeeBucketListInTheConsole", - "Action": [ - "s3:ListAllMyBuckets", - "s3:GetBucketLocation" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:s3:::*" - ] - }, - { - "Sid": "AllowPutActionInBucket", - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:PutObject", - "s3:ListBucket" - ], - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:aws:s3:::", - { - "Ref": "PyPiS3Bucket" - }, - "/" - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:aws:s3:::", - { - "Ref": "PyPiS3Bucket" - }, - "/*" - ] - ] - } - ] - } - ] - } - } - } - } -} \ No newline at end of file diff --git a/cloudformation/s3-pypi.json b/cloudformation/s3-pypi.json deleted file mode 100644 index 13087b0..0000000 --- a/cloudformation/s3-pypi.json +++ /dev/null @@ -1,246 +0,0 @@ -{ - "AWSTemplateFormatVersion": "2010-09-09", - "Description": "s3pypi setup - Visit https://novemberfive.co.", - "Mappings": { - "RegionMap": { - "us-east-1": { - "WebsiteEndpoint": "s3-website-us-east-1.amazonaws.com" - }, - "us-west-2": { - "WebsiteEndpoint": "s3-website-us-west-2.amazonaws.com" - }, - "us-west-1": { - "WebsiteEndpoint": "s3-website-us-west-1.amazonaws.com" - }, - "eu-west-1": { - "WebsiteEndpoint": "s3-website-eu-west-1.amazonaws.com" - }, - "eu-central-1": { - "WebsiteEndpoint": "s3-website.eu-central-1.amazonaws.com" - }, - "ap-southeast-1": { - "WebsiteEndpoint": "s3-website-ap-southeast-1.amazonaws.com" - }, - "ap-northeast-1": { - "WebsiteEndpoint": "s3-website-ap-northeast-1.amazonaws.com" - }, - "ap-southeast-2": { - "WebsiteEndpoint": "s3-website-ap-southeast-2.amazonaws.com" - }, - "ap-northeast-2": { - "WebsiteEndpoint": "s3-website.ap-northeast-2.amazonaws.com" - }, - "sa-east-1": { - "WebsiteEndpoint": "s3-website-sa-east-1.amazonaws.com" - } - } - }, - "Outputs": { - "CNAMERecordValue": { - "Description": "The value of the CNAME or alias record of the configured (sub)domain.", - "Value": { - "Fn::GetAtt": [ - "PyPiCloudfrontDistribution", - "DomainName" - ] - } - } - }, - "Parameters": { - "DomainName": { - "Description": "The (sub)domain that you want to use for your PyPi repository (e.g. pypi.yourcompany.com)", - "Type": "String" - }, - "ServerCertificateId": { - "Description": "ID of the server certificate for the domain name (e.g. ASCAXXXXXXXXXXXXXXXXX)", - "Type": "String" - } - }, - "Resources": { - "PyPiS3Bucket": { - "Type": "AWS::S3::Bucket", - "Properties": { - "BucketName": { - "Ref": "DomainName" - }, - "WebsiteConfiguration": { - "IndexDocument": "index.html" - } - } - }, - "PyPiS3BucketBucketPolicy": { - "Type": "AWS::S3::BucketPolicy", - "Properties": { - "Bucket": { - "Ref": "PyPiS3Bucket" - }, - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetObject" - ], - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:aws:s3:::", - { - "Ref": "PyPiS3Bucket" - }, - "/*" - ] - ] - }, - "Principal": "*", - "Condition": { - "IpAddress": { - "aws:SourceIp": [ - "52.84.0.0/15", - "54.182.0.0/16", - "54.192.0.0/16", - "54.230.0.0/16", - "54.239.128.0/18", - "54.239.192.0/19", - "54.240.128.0/18", - "204.246.164.0/22", - "204.246.168.0/22", - "204.246.174.0/23", - "204.246.176.0/20", - "205.251.254.0/24", - "205.251.252.0/23", - "205.251.250.0/23", - "205.251.249.0/24", - "205.251.192.0/19", - "216.137.32.0/19" - ] - } - } - } - ] - } - } - }, - "PyPiCloudfrontDistribution": { - "Type": "AWS::CloudFront::Distribution", - "Properties": { - "DistributionConfig": { - "Aliases": [ - { - "Ref": "DomainName" - } - ], - "Comment": "Created by s3pypi", - "DefaultCacheBehavior": { - "AllowedMethods": [ - "GET", - "HEAD" - ], - "ForwardedValues": { - "QueryString": "true" - }, - "TargetOriginId": "PyPiS3BucketOrigin", - "ViewerProtocolPolicy": "https-only" - }, - "Enabled": "true", - "Origins": [ - { - "CustomOriginConfig": { - "HTTPPort": "80", - "HTTPSPort": "443", - "OriginProtocolPolicy": "http-only" - }, - "DomainName": { - "Fn::Join": [ - ".", - [ - { - "Ref": "PyPiS3Bucket" - }, - { - "Fn::FindInMap": [ - "RegionMap", - { - "Ref": "AWS::Region" - }, - "WebsiteEndpoint" - ] - } - ] - ] - }, - "Id": "PyPiS3BucketOrigin" - } - ], - "PriceClass": "PriceClass_100", - "ViewerCertificate": { - "IamCertificateId": { - "Ref": "ServerCertificateId" - }, - "MinimumProtocolVersion": "TLSv1", - "SslSupportMethod": "sni-only" - } - } - } - }, - "PublishS3PyPiPackages": { - "Type": "AWS::IAM::ManagedPolicy", - "Properties": { - "Description": "Policy for updating packages in S3", - "Path": "/", - "PolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowUserToSeeBucketListInTheConsole", - "Action": [ - "s3:ListAllMyBuckets", - "s3:GetBucketLocation" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:s3:::*" - ] - }, - { - "Sid": "AllowPutActionInBucket", - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:PutObject", - "s3:ListBucket" - ], - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:aws:s3:::", - { - "Ref": "PyPiS3Bucket" - }, - "/" - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:aws:s3:::", - { - "Ref": "PyPiS3Bucket" - }, - "/*" - ] - ] - } - ] - } - ] - } - } - } - } -} \ No newline at end of file diff --git a/pypis3/__init__.py b/pypis3/__init__.py new file mode 100644 index 0000000..5f1c073 --- /dev/null +++ b/pypis3/__init__.py @@ -0,0 +1,2 @@ +__prog__ = 'pypis3' +__version__ = u'0.1.0' diff --git a/s3pypi/cli.py b/pypis3/cli.py similarity index 69% rename from s3pypi/cli.py rename to pypis3/cli.py index 197b4f6..51043b8 100644 --- a/s3pypi/cli.py +++ b/pypis3/cli.py @@ -1,19 +1,18 @@ #!/usr/bin/env python import argparse -from s3pypi import __prog__, __version__ -from s3pypi.package import Package -from s3pypi.storage import S3Storage +from pypis3 import __prog__, __version__ +from pypis3.package import Package +from pypis3.storage import S3Storage -__author__ = 'Matteo De Wint' -__copyright__ = 'Copyright 2016, November Five' +__author__ = 'Jamie Cressey' +__copyright__ = 'Copyright 2016, Jamie Cressey' __license__ = 'MIT' def main(): p = argparse.ArgumentParser(prog=__prog__, version=__version__) p.add_argument('--bucket', required=True, help='S3 bucket') - p.add_argument('--secret', help='S3 secret') p.add_argument('--url', help='Custom URL', default=None) p.add_argument( '--no-wheel', @@ -23,7 +22,7 @@ def main(): args = p.parse_args() package = Package.create(args.wheel) - storage = S3Storage(args.bucket, args.secret, args.url) + storage = S3Storage(args.bucket, args.url) index = storage.get_index(package) index.packages.discard(package) diff --git a/s3pypi/package.py b/pypis3/package.py similarity index 95% rename from s3pypi/package.py rename to pypis3/package.py index 0ea990c..e52af09 100644 --- a/s3pypi/package.py +++ b/pypis3/package.py @@ -6,10 +6,10 @@ from jinja2 import Environment, PackageLoader -from s3pypi import __prog__ +from pypis3 import __prog__ -__author__ = 'Matteo De Wint' -__copyright__ = 'Copyright 2016, November Five' +__author__ = 'Jamie Cressey' +__copyright__ = 'Copyright 2016, Jamie Cressey' __license__ = 'MIT' diff --git a/s3pypi/storage.py b/pypis3/storage.py similarity index 86% rename from s3pypi/storage.py rename to pypis3/storage.py index 6fa9a6b..875cf08 100644 --- a/s3pypi/storage.py +++ b/pypis3/storage.py @@ -2,20 +2,19 @@ import boto3 -from s3pypi.package import Index +from pypis3.package import Index -__author__ = 'Matteo De Wint' -__copyright__ = 'Copyright 2016, November Five' +__author__ = 'Jamie Cressey' +__copyright__ = 'Copyright 2016, Jamie Cressey' __license__ = 'MIT' class S3Storage(object): """Abstraction for storing package archives and index files in an S3 bucket.""" - def __init__(self, bucket, secret=None, url=None): + def __init__(self, bucket, url=None): self.s3 = boto3.client('s3') self.bucket = bucket - self.secret = secret self.url = url diff --git a/s3pypi/templates/index.html.j2 b/pypis3/templates/index.html.j2 similarity index 100% rename from s3pypi/templates/index.html.j2 rename to pypis3/templates/index.html.j2 diff --git a/s3pypi/__init__.py b/s3pypi/__init__.py deleted file mode 100644 index a7c9a27..0000000 --- a/s3pypi/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -__prog__ = 's3pypi' -__version__ = u'0.0.9' diff --git a/setup.py b/setup.py index 0e977e4..290d67b 100644 --- a/setup.py +++ b/setup.py @@ -1,18 +1,18 @@ from setuptools import setup, find_packages -from s3pypi import __prog__, __version__ +from pypis3 import __prog__, __version__ setup( name=__prog__, version=__version__, - description='CLI tool for creating a Python Package Repository in an S3 bucket', + description='CLI tool for creating a private Python Package Repository backed entirely by AWS S3 storage', - author='Ruben Van den Bossche, Matteo De Wint', - author_email='ruben@novemberfive.co, matteo@novemberfive.co', - url='https://github.com/novemberfiveco/s3pypi', - download_url='https://github.com/novemberfiveco/s3pypi/tarball/' + __version__, + author='Jamie Cressey', + author_email='jamiecressey89@gmail.com', + url='https://github.com/JamieCressey/pypis3', + download_url='https://github.com/jamiecressey/pypis3/tarball/' + __version__, packages=find_packages(), package_data={__prog__: ['templates/*.j2']}, diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py deleted file mode 100644 index 584991b..0000000 --- a/tests/unit/test_storage.py +++ /dev/null @@ -1,6 +0,0 @@ -from s3pypi.storage import S3Storage - - -def test_secret(): - storage = S3Storage('appstrakt-pypi', 'secret') - assert 'secret' in storage.url