Commit 751f9088 authored by Iosif Peterfi's avatar Iosif Peterfi
Browse files

Removed repo links for cert-issuer and cert-tools

parent 6f334f41
.cache
.tox
*egg-info
venv
pk
*.pyc
.DS_Store
venv*/
virtual\ usb/
secrets.py
__pycache__
pk
certs_dbdump.csv
data/unsigned_certificates/
data/signed_certificates/
data/blockchain_certificates/
data/work/*
!data/unsigned_certificates/.*
!data/signed_certificates/.*
!data/blockchain_certificates/.*
!data/work/.*
data*/*
dist/*
build/*
requirements.txt
.idea/*
*.ini*
pk_issuer.txt
blockcerts testnet bitcoin address.txt
.vscode/*
.cache/*
.tox/*
*.egg-info/*
docs/_build/*
site/*
priv*.txt
place holder for htaccess
sudo: false
language: python
python:
- "3.6"
install: pip install tox-travis
script: tox
FROM seegno/bitcoind:0.13-alpine
MAINTAINER Kim Duffy "kimhd@mit.edu"
COPY . /cert-issuer
COPY conf_regtest.ini /etc/cert-issuer/conf.ini
RUN apk add --update \
bash \
ca-certificates \
curl \
gcc \
gmp-dev \
libffi-dev \
libressl-dev \
linux-headers \
make \
musl-dev \
python \
python3 \
python3-dev \
tar \
&& python3 -m ensurepip \
&& pip3 install --upgrade pip setuptools \
&& mkdir -p /etc/cert-issuer/data/unsigned_certificates \
&& mkdir /etc/cert-issuer/data/blockchain_certificates \
&& mkdir ~/.bitcoin \
&& echo $'rpcuser=foo\nrpcpassword=bar\nrpcport=8332\nregtest=1\nrelaypriority=0\nrpcallowip=127.0.0.1\nrpcconnect=127.0.0.1\n' > /root/.bitcoin/bitcoin.conf \
&& pip3 install /cert-issuer/. \
&& rm -r /usr/lib/python*/ensurepip \
&& rm -rf /var/cache/apk/* \
&& rm -rf /root/.cache \
&& sed -i.bak s/==1\.0b1/\>=1\.0\.2/g /usr/lib/python3.*/site-packages/merkletools-1.0.2-py3.*.egg-info/requires.txt
ENTRYPOINT bitcoind -daemon && bash
FROM ubuntu:18.04
maintainer yancy ribbens "yribbens@credly.com"
RUN apt-get update -qq && apt-get install -y git wget build-essential libtool autotools-dev automake pkg-config libssl-dev libevent-dev bsdmainutils python3 libboost-all-dev libminiupnpc-dev libzmq3-dev python3-pip locales vim python3.6 python3.6-dev uwsgi uwsgi-src uuid-dev libcap-dev libpcre3-dev python-pip python-dev nginx netcat
# Checkout bitcoin source
RUN mkdir /bitcoin-source
WORKDIR /bitcoin-source
RUN git clone https://github.com/bitcoin/bitcoin.git
# Install Berkley Database
RUN wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz
RUN tar -xvf db-4.8.30.NC.tar.gz
WORKDIR /bitcoin-source/db-4.8.30.NC/build_unix
RUN mkdir -p build
RUN BDB_PREFIX=$(pwd)/build
RUN ../dist/configure --disable-shared --enable-cxx --with-pic --prefix=$BDB_PREFIX
RUN make install
# install bitcoin 0.16.3
WORKDIR /bitcoin-source/bitcoin
RUN git checkout tags/v0.16.3
RUN ./autogen.sh
RUN ./configure CPPFLAGS="-I${BDB_PREFIX}/include/ -O2" LDFLAGS="-L${BDB_PREFIX}/lib/" --without-gui
RUN make
RUN make install
# configure bitcoin network
ARG NETWORK=regtest
ARG RPC_USER=foo
ARG RPC_PASSWORD=bar
RUN mkdir -p ~/.bitcoin
RUN echo "rpcuser=${RPC_USER}\nrpcpassword=${RPC_PASSWORD}\n${NETWORK}=1\nrpcport=8332\nrpcallowip=127.0.0.1\nrpcconnect=127.0.0.1\n" > /root/.bitcoin/bitcoin.conf
# default to UTF8 character set
ENV LANGUAGE en_US.UTF-8
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_TYPE en_US.UTF-8
RUN locale-gen en_US.UTF-8
# configure cert-issuer
ARG ISSUER=<issuing-address>
COPY . /cert-issuer
COPY conf_regtest.ini /etc/cert-issuer/conf.ini
RUN mkdir -p /etc/cert-issuer/
WORKDIR /cert-issuer
RUN pip3 install cert-issuer
RUN pip3 install -r requirements.txt
RUN sed -i.bak "s/<issuing-address>/$ISSUER/g" /etc/cert-issuer/conf.ini
# configure wsgi
ENV PYTHON python3.6
WORKDIR /root
RUN uwsgi --build-plugin "/usr/src/uwsgi/plugins/python python36"
RUN cp /root/python36_plugin.so /cert-issuer
RUN chmod 644 /cert-issuer/python36_plugin.so
RUN virtualenv -p python3 venv
RUN pip3 install uwsgi flask
# configure nginx
ARG SERVER=<server-name>
EXPOSE 80
WORKDIR /cert-issuer
COPY cert_issuer_site /etc/nginx/sites-available
RUN ln -s /etc/nginx/sites-available/cert_issuer_site /etc/nginx/sites-enabled
RUN sed -i.bak "s/<server-name>/$SERVER/g" /etc/nginx/sites-available/cert_issuer_site
RUN sed -i.bak "s/# server_names_hash_bucket_size 64/server_names_hash_bucket_size 128/g" /etc/nginx/nginx.conf
RUN chmod +x start-cert-issuer.sh
ENTRYPOINT bitcoind -daemon && service nginx start && uwsgi --ini wsgi.ini
FROM ubuntu:18.04
maintainer yancy ribbens "yribbens@credly.com"
VOLUME /home/.bitcoin
RUN apt-get update -qq && apt-get install -y git \
wget \
build-essential \
libtool \
autotools-dev \
automake \
pkg-config \
libssl-dev \
libevent-dev \
bsdmainutils \
python3 \
libboost-all-dev \
libminiupnpc-dev \
libzmq3-dev \
python3-pip \
locales \
vim \
python3.6 \
python3.6-dev \
uwsgi \
uwsgi-src \
uuid-dev \
libcap-dev \
libpcre3-dev \
python-pip \
python-dev \
nginx \
apache2-utils
# Checkout bitcoin source
RUN mkdir /bitcoin-source
WORKDIR /bitcoin-source
RUN git clone https://github.com/bitcoin/bitcoin.git
# Install Berkley Database
RUN wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz
RUN tar -xvf db-4.8.30.NC.tar.gz
WORKDIR /bitcoin-source/db-4.8.30.NC/build_unix
RUN mkdir -p build
RUN BDB_PREFIX=$(pwd)/build
RUN ../dist/configure --disable-shared --enable-cxx --with-pic --prefix=$BDB_PREFIX
RUN make install
# install bitcoin 0.16.3
WORKDIR /bitcoin-source/bitcoin
RUN git checkout tags/v0.16.3
RUN ./autogen.sh
RUN ./configure CPPFLAGS="-I${BDB_PREFIX}/include/ -O2" LDFLAGS="-L${BDB_PREFIX}/lib/" --without-gui
RUN make
RUN make install
# configure bitcoin network
ARG NETWORK=regtest
ARG RPC_USER=foo
ARG RPC_PASSWORD=bar
RUN mkdir -p ~/.bitcoin
RUN echo "rpcuser=${RPC_USER}\nrpcpassword=${RPC_PASSWORD}\n${NETWORK}=1\nrpcport=8332\nrpcallowip=127.0.0.1\nrpcconnect=127.0.0.1\n" > /root/.bitcoin/bitcoin.conf
# default to UTF8 character set
ENV LANGUAGE en_US.UTF-8
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_TYPE en_US.UTF-8
RUN locale-gen en_US.UTF-8
# configure cert-issuer
ARG ISSUER=<issuing-address>
ARG KEY=cert.key
ARG CRT=cert.crt
COPY . /cert-issuer
COPY conf_regtest.ini /etc/cert-issuer/conf.ini
COPY $CRT /etc/ssl/certs/$CRT
COPY $KEY /etc/ssl/private/$KEY
RUN mkdir -p /etc/cert-issuer/
WORKDIR /cert-issuer
RUN pip3 install cert-issuer
RUN pip3 install -r requirements.txt
RUN sed -i.bak "s/<issuing-address>/$ISSUER/g" /etc/cert-issuer/conf.ini
# configure wsgi
ENV PYTHON python3.6
WORKDIR /root
RUN uwsgi --build-plugin "/usr/src/uwsgi/plugins/python python36"
RUN cp /root/python36_plugin.so /cert-issuer
RUN chmod 644 /cert-issuer/python36_plugin.so
RUN virtualenv -p python3 venv
RUN pip3 install uwsgi flask
# configure nginx
ARG SERVER=<server-name>
EXPOSE 443
WORKDIR /cert-issuer
COPY cert_issuer_tls_site /etc/nginx/sites-available
COPY .htpasswd /etc/nginx/htpasswd
RUN chmod -R 640 /etc/nginx/htpasswd
RUN chown :www-data /etc/nginx/htpasswd
RUN ln -s /etc/nginx/sites-available/cert_issuer_tls_site /etc/nginx/sites-enabled
RUN sed -i.bak "s/<server-name>/$SERVER/g" /etc/nginx/sites-available/cert_issuer_tls_site
RUN sed -i.bak "s/<cert-crt-name>/$CRT/g" /etc/nginx/sites-available/cert_issuer_tls_site
RUN sed -i.bak "s/<cert-key-name>/$KEY/g" /etc/nginx/sites-available/cert_issuer_tls_site
RUN sed -i.bak "s/# server_names_hash_bucket_size 64/server_names_hash_bucket_size 128/g" /etc/nginx/nginx.conf
ENTRYPOINT bitcoind -daemon && service nginx start && uwsgi --ini wsgi.ini
FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7
EXPOSE 80
COPY ./app /app/app
COPY ./cert_issuer /app/cert_issuer
COPY ./sample_data /app/sample_data
COPY ethereum_smart_contract_requirements.txt .
COPY conf.ini .
RUN pip install -r ethereum_smart_contract_requirements.txt
The MIT License (MIT)
Copyright (c) 2016 Media Lab Learning Initiative
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
include LICENSE *.md *.txt conf_regtest.ini
include cert_issuer/blockchain_handlers/ethereum_sc/data/ens_registry_abi.json
include cert_issuer/blockchain_handlers/ethereum_sc/data/ens_resolver_abi.json
\ No newline at end of file
[![Build Status](https://travis-ci.org/blockchain-certificates/cert-issuer.svg?branch=master)](https://travis-ci.org/blockchain-certificates/cert-issuer)
[![PyPI version](https://badge.fury.io/py/cert-issuer.svg)](https://badge.fury.io/py/cert-issuer)
# cert-issuer
The cert-issuer project issues blockchain certificates by creating a transaction from the issuing institution to the
recipient on the Ethereum blockchain that includes the hash of the certificate itself.
Currently, we support issuing to the bloxberg blockchain network using the BLIPS-2 Standard for Research Object Certification.
The instructions below pertain mainly to using the cert-issuer via CLI. For the API interface, please view our cert-api repository.
## Generating unsigned certificates
In order to create unsigned certificates, you must first follow the instructions in our [cert-tools](https://github.com/bloxberg-org/cert-tools) repository. The below instructions assume you have already a batch of unsigned certificates generated.
## Deploying smart contract
Before issuing certificates, you must also ensure that an associated smart contract is deployed to the bloxberg blockchain that your address has control over. Please follow the instructions located in [cert-deployer](https://github.com/bloxberg-org/cert-deployer) to deploy the contract.
### Issuing certificates
1. Create a python environment. [Recommendations](https://github.com/bloxberg-org/cert-issuer/blob/master/docs/virtualenv.md)
1. Run the following command in the terminal:
```
python setup.py install experimental --blockchain=ethereum_smart_contract
```
1. Copy the `conf_template.ini` file and rename as `conf.ini`.
1. Make sure you have enough ETH in your issuing address.
a. Go to https://myetherwallet.com and create a public address and private key. Make sure to store the private key in a secure location.
b. Go to https://faucet.bloxberg.org and enter your created public address to receive bergs.
1. Configure the unsigned_certificates_dir variable in the conf.ini file to the unsigned certificates directory. Alternatively, you can also specify the directory created from the [cert-tools](https://github.com/bloxberg-org/cert-tools) repository to avoid copying the unsigned certificates.
```
# To use a sample unsigned certificate as follows:
cp /cert-issuer/examples/data-testnet/unsigned_certificates/3bc1a96a-3501-46ed-8f75-49612bbac257.json /cert-issuer/data/unsigned_certificates/
```
1. Further configure the settings for cert-issuer in `conf.ini`. The private key file [should be](https://eth-account.readthedocs.io/en/latest/eth_account.html#eth_account.account.Account.from_key) a raw private key: a hex str, bytes or int
1. Lastly, we can issue the certificates by running
```
python cert_issuer -c conf.ini
```
# How batch issuing works
While it is possible to issue one certificate with one Ethereum transaction, it is far more efficient to use one Ethereum transaction to issue a batch of certificates.
The issuer builds a Merkle tree of certificate hashes and registers the Merkle root as the cryptographic identifier in an Ethereum smart contract.
Suppose the batch contains `n` certificates, and certificate `i` contains recipient `i`'s information. The issuer hashes each certificate and combines them into a Merkle tree:
![](img/merkle.png)
The root of the Merkle tree, which is a 256-bit hash, is issued on the bloxberg blockchain.
The Blockchain Certificate given to recipient `i` contains a [2019 Merkle Proof Signature Suite](https://w3c-ccg.github.io/lds-merkle-proof-2019/)-formatted signature, proving that certificate `i` is contained in the Merkle tree.
![](img/blockchain_certificate_components.png)
This receipt contains:
- The Ethereum transaction ID storing the Merkle root
- The expected Merkle root on the blockchain
- The expected hash for recipient `i`'s certificate
- The Merkle path from recipient `i`'s certificate to the Merkle root, i.e. the path highlighted in orange above. `h_i -> … -> Merkle root`
The [verification process](https://github.com/blockchain-certificates/cert-verifier-js#verification-process) performs computations to check that:
- The hash of certificate `i` matches the value in the receipt
- The Merkle path is valid
- The Merkle root stored on the blockchain matches the value in the receipt
These steps establish that the certificate has not been tampered with since it was issued.
## Hashing a certificate
The Blockchain Certificate JSON contents without the `signature` node is the certificate that the issuer created. This is the value needed to hash for comparison against the receipt. Because there are no guarantees about ordering or formatting of JSON, first canonicalize the certificate (without the `signature`) against the JSON LD schema. This allows us to obtain a deterministic hash across platforms.
The detailed steps are described in the [verification process](https://github.com/blockchain-certificates/cert-verifier-js#verification-process).
## What should be in a batch?
How a batch is defined can vary, but it should be defined such that it changes infrequently. For example, “2016 MIT grads” would be preferred over “MIT grads” (the latter would have to be updated every year).
## Prerequisites
Decide which chain (Bitcoin or Ethereum) to issue to and follow the steps. The bitcoin chain is currently best supported by the Blockcerts libraries. Follow the steps for the chosen chain.
### Install cert-issuer
By default, cert-issuer issues to the bloxberg blockchain. Run this setup to issue to the standard chain:
To issue to the ethereum blockchain using the smart contract backend, run the following:
```
python setup.py install experimental --blockchain=ethereum_smart_contract
```
### Create an Ethereum issuing address
Currently, we support issuing to the bloxberg blockchain network using the BLIPS-2 Standard for Research Object Certification.
**These steps involve storing secure information on a USB. Do not plug in this USB when your computer's wifi is on.**
1. Create issuing address on Myetherwallet
- Go to https://www.myetherwallet.com/.
- For the best security turn off your connection to the internet when you are on the create wallet page.
2. Go through the create wallet process
- Store the private key in a secure location
- Copy the public key to the `issuing_address` value in conf.ini
# Unit tests
This project uses tox to validate against several python environments.
1. Ensure you have an python environment. [Recommendations](docs/virtualenv.md)
2. Run tests
```
./run_tests.sh
```
# Class design
## Core issuing classes
![](img/issuer_main_classes.png)
The `Issuer` api is quite simple; it relies on `CertificateHandler`s and `Transaction Handler`s to do the work of
extracting the data to issue on the blockchain, and handling the blockchain transaction, respectively.
`CertificateBatchHandler` manages the certificates to issue on the blockchain. It ensures that all accessors iterate
certificates in a predictable order. This is critical because the Merkle Proofs must be associated with the correct
certificate. Python generators are used here to help keep the memory footprint low while reading from files.
- `prepare_batch`
- performs the preparatory steps on certificates in the batch, including validation of the schema and forming the
data that will go on the blockchain. Certificate-level details are handled by `CertificateHandler`s
- returns the hex byte array that will go on the blockchain
- `finish_batch` ensures each certificate is updated with the blockchain transaction information (and proof in general)
`CertificateHandler` is responsible for reading from and updating a specific certificate (identified by certificate_metadata).
It is used exclusively by `CertificateBatchHandler` to handle certificate-level details:
- `validate`: ensure the certificate is well-formed
- `sign`: (currently unused)
- `get_byte_array_to_issue`: return byte array that will be hashed, hex-digested and added to the Merkle Tree
- `add_proof`: associate a a proof with a certificate (in the current implementation, the proof is embedded in the file)
`TransactionHandler` deals with putting the data on the blockchain. Currently only a Bitcoin implementation exists
## Signing and secret management
![](img/signing_classes.png)
Finalizable signer is a convenience class allowing use of python's `with` syntax. E.g.:
```
with FinalizableSigner(secret_manager) as fs:
fs.sign_message(message_to_sign)
```
SecretManagers ensure the secret key (wif) is loaded into memory for signing. FileSecretManager is the only current
implemenation.
## Merkle tree generator
![](img/merkle_tree_generator.png)
Handles forming the Merkle Tree, returning the data to put on the blockchain, and returning a python generator of the
proofs.
This class structure is intended to be general-purpose to allow other implementations. (Do this carefully if at all.)
# Advanced setup
- [Installing and using a local bitcoin node](docs/bitcoind.md)
# Publishing To Pypi
- Create an account for [pypi](https://pypi.org) & [pypi test](https://test.pypi.org)
- Install [twine](github.com/pypa/twine) - `pip install twine`
- Increment version in `__init__.py`
- Remove current items in dist - `rm -rf dist/*`
- Build cert-issuer - `python setup.py install`
- Build sdist - `python setup.py sdist`
- Run pypi test upload - `twine upload --repository-url https://test.pypi.org/legacy/ dist/*`
- Upload to pypi - `twine upload --repository-url https://upload.pypi.org/legacy/ dist/*`
# Examples
The files in examples/data-testnet contain results of previous runs.
# FAQs
## Checking transaction status
- Ethereum Bloxberg: https://blockexplorer.bloxberg.org and search for the tx id.
## Mac scrypt problems
If your install on Mac is failing with a message like the following, try the [workaround described in this thread](https://github.com/ethereum/pyethereum/issues/292).
```
fatal error: 'openssl/aes.h'
file not found
#include <openssl/aes.h>
```
# Contact
Contact us at info@bloxberg.org for more information.
#!/usr/bin/python3
import json
from flask import Flask, jsonify, request, abort
from subprocess import call
import cert_issuer.config
from cert_issuer.blockchain_handlers import ethereum_sc
import cert_issuer.issue_certificates
app = Flask(__name__)
config = None
def get_config():
global config
if config == None:
config = cert_issuer.config.get_config()
return config
@app.route('/cert_issuer/api/v1.0/issue', methods=['POST'])
def issue():
config = get_config()
certificate_batch_handler, transaction_handler, connector = \
ethereum_sc.instantiate_blockchain_handlers(config)
#Removed File mode from ethereum_sc
#bitcoin.instantiate_blockchain_handlers(config, False)
certificate_batch_handler.set_certificates_in_batch(request.json)
cert_issuer.issue_certificates.issue(config, certificate_batch_handler, transaction_handler)
return json.dumps(certificate_batch_handler.proof)
if __name__ == '__main__':
app.run(host='0.0.0.0')
from typing import List, Optional
from functools import lru_cache
from fastapi import Depends, FastAPI
from cert_tools import instantiate_v3_alpha_certificate_batch, create_v3_alpha_certificate_template
from pydantic import BaseModel
import configargparse
import os
app = FastAPI()
@app.get("/")
def read_root():
return {"Hello": "World"}
class Batch(BaseModel):
publicKey: str
recipient_name: Optional[str]
email: Optional[str]
SHA256: List[str]
class Config:
schema_extra = {
"example": {
"publicKey": "0x69575606E8b8F0cAaA5A3BD1fc5D032024Bb85AF",
"recipient_name": "Albert Einstein",
"email": "einstein@mpg.de",
"SHA256": ["0x0e4ded5319861c8daac00d425c53a16bd180a7d01a340a0e00f7dede40d2c9f6", "0xfda3124d5319861c8daac00d425c53a16bd180a7d01a340a0e00f7dede40d2c9f6"],
}
}
@app.post("/createUnsignedCertificateBatch")
def createUnsignedCertificate(batch: Batch):
#create_v3_alpha_certificate_template(recipient_name, email)
conf = create_v3_alpha_certificate_template.get_config()
create_v3_alpha_certificate_template.write_certificate_template(conf, batch.recipient_name, batch.email)
conf_instantiate = instantiate_v3_alpha_certificate_batch.get_config()
instantiate_v3_alpha_certificate_batch.instantiate_batch(conf_instantiate, batch.publicKey, batch.recipient_name, batch.email, batch.SHA256)
return {"Unsigned Certificate Created!"}
@app.post("/createCertificateBatch")
def createCertificateBatch(batch: Batch):
conf = instantiate_v3_alpha_certificate_batch.get_config()
instantiate_v3_alpha_certificate_batch.instantiate_batch(conf, batch.publicKey, batch.recipient_name, batch.email, batch.SHA256)
return {"Batch Created!"}
python-bitcoinlib>=0.10.1
\ No newline at end of file
#!/usr/bin/env python3
import os.path
import sys
import cProfile
PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if __package__ is None and not hasattr(sys, 'frozen'):
path = os.path.realpath(os.path.abspath(__file__))
sys.path.insert(0, os.path.dirname(os.path.dirname(path)))
def cert_issuer_main(args=None):