Create vault
user.
$ sudo useradd
-r -g daemon -d /opt/vault -m -s /sbin/nologin -c "Vault user" vault
$ cd /tmp
$ wget https://releases.hashicorp.com/vault/0.10.1/vault_0.10.1_linux_amd64.zip
$ sudo mkdir -p
/opt/vault/bin
$ cd
/opt/vault/bin
$ sudo unzip /tmp/vault_0.10.1_linux_amd64.zip
$ sudo ln -s vault
/usr/local/bin/vault
$ sudo chown
-R vault:root /opt/vault
$ sudo chmod
-R 755 /opt/vault
$ sudo mkdir
/opt/vault/conf
Change to
vault user
$ sudo su -
vault -s /bin/bash
$ cd
/opt/vault/conf
Create
configuration file vault-conf.hcl
ui = true
listener
"tcp" {
address = "0.0.0.0:8200"
tls_disable = 1
}
storage
"s3" {
bucket = "vault"
region = "us-east-1"
}
disable_mlock=true
Start Vault
$ nohup vault
server -config=/opt/vault/conf/vault-conf.hcl > /var/log/vault/vault-debug.log
2>&1 &
$ jobs
Verify Server
is running
$ vault
status
$ export
VAULT_ADDR=http://0.0.0.0:8200
Intialize
Vault
$ vault
operator init
Copy keys to
/opt/vault/keys/vault_keys.txt
This will
output five Unseal Keys, which need to be used to unseal the Vault. The
initialization defaults to minimum of three tokens to allow unseal. There will
also be Initial Root Token in the output. It should only be used for initial
configuration, while recurring operations should be done by policy-constrained
tokens.
Unseal Vault
with three different keys.
$ vault operator
unseal
Authenticate with
Vault as Initial Root Token to perform any operations on an unsealed Vault.
$ vault login
Enable Audit
Logging
$ vault audit
enable file file_path=/var/log/vault/vault-audit.log
List enabled
auth methods
$ vault auth
list
Path Type
Description
---- ----
-----------
token/ token
token based credentials
Integrate Active Directory
Enable LDAP
auth method.
$ vault auth
enable ldap
Success!
Enabled ldap auth method at: ldap/
$ vault write auth/ldap/config \
binddn="cn=Lkup_user,ou=Standard,ou=Service
Accounts,ou=Accounts,ou=abc,ou=External,ou=xyz,dc=example,dc=com"
\
bindpass='passwd' \
deny_null_bind=true \
discoverdn=false \
groupattr="cn" \
groupdn="OU=Standard,OU=Groups,OU=Prod,OU=abc,OU=External,OU=xyz,DC=example,DC=com"
\
groupfilter="(&(objectClass=group)(member:1.2.840.113556.1.4.1941:={{.UserDN}}))"
\
insecure_tls=true \
starttls=false \
url="ldap://ldap.example.com" \
userattr="samaccountname" \
userdn="OU=Users,OU=Accounts,OU=xyz,DC=example,DC=com"
# cat admin-policy.hcl
# List existing auth methods
path "sys/auth"
{
capabilities = ["read"]
}
# Manage auth methods broadly across Vault
path "sys/auth/*"
{
capabilities = ["create", "read", "update", "delete", "list", "sudo"]
}
# List existing policies
path "sys/policy"
{
capabilities = ["read"]
}
path "sys/policies"
{
capabilities = ["read"]
}
# Create and manage ACL policies broadly across Vault
path "sys/policy/*"
{
capabilities = ["create", "read", "update", "delete", "list", "sudo"]
}
path "sys/policies/*"
{
capabilities = ["create", "read", "update", "delete", "list", "sudo"]
}
# List existing KV secrets v2
path "secret"
{
capabilities = ["read"]
}
# Create and manage KV secrets engine v2 broadly across Vault.
path "secret/*"
{
capabilities = ["create", "read", "update", "delete", "list", "sudo"]
}
# List existing databases
path "database"
{
capabilities = ["read"]
}
# Create and manage databases broadly across Vault.
path "database/*"
{
capabilities = ["create", "read", "update", "delete", "list", "sudo"]
}
# List existing identities
path "identity"
{
capabilities = ["read"]
}
# Create and manage identities broadly across Vault.
path "identity/*"
{
capabilities = ["create", "read", "update", "delete", "list", "sudo"]
}
# Read health checks
path "sys/health"
{
capabilities = ["read", "sudo"]
}
# To perform capabilities of token
path "sys/capabilities"
{
capabilities = ["create", "update"]
}
path "sys/capabilities-self"
{
capabilities = ["create", "update"]
}
Create admin policy.
$ vault policy write admin admin-policy.hcl
Map LDAP group to policies. For example, map vault-admin group to admin policy.
$ vault write auth/ldap/groups/vault-admin policies=admin
-----------------------------------------------------------------------------------------
AD Group Vault Path
-----------------------------------------------------------------------------------------
vault-admin secret/*
vault-admin-<env>-mysql secret/<env>/MySQL/<key-name>
vault-admin-<env>-postgres secret/<env>/PostgreSQL/<key-name>
vault-admin-<env>-oracle secret/<env>/Oracle/<key-name>
vault-admin-<env>-hadoop secret/<env>/Hadoop/<key-name>
vault-read-<env>-mysql secret/<env>/MySQL/<key-name>
vault-read-<env>-postgres secret/<env>/PostgreSQL/<key-name>
vault-read-<env>-oracle secret/<env>/Oracle/<key-name>
vault-read-<env>-hadoop secret/<env>/Hadoop/<key-name>
<env> - prod/impl/test/dev
<key-name> - oracle_instance or mysql_name etc
List enabled secret
backends
$ vault
secrets list
Path Type Description
---- ---- -----------
cubbyhole/ cubbyhole
per-token private secret storage
identity/ identity identity store
secret/ kv key/value secret storage
sys/ system system endpoints used for control,
policy and debugging
Disable KV
secrets engine – version 1
$ vault
secrets disable secret
Enable KV
secrets engine - version 2
$ vault secrets
enable -path=secret -description="key/value secret storage v2"
-version=2 kv
List existing
keys
$ vault kv
list secret
$ vault kv put secret/prod/Oracle/<instance_name> username=<user> password=<passwd>
$ curl -s -H "X-Vault-Token: $VAULT_TOKEN" -x GET $VAULT_ADDR/v1/secret/data/prod/Oracle/<instance_name> | jq '.data.data.username,.data.data.password';
Configure AppRole for Streamsets
Enable AppRole
auth method.
$ vault auth
enable approle
Success!
Enabled approle auth method at: approle/
$ vault auth
list
Path Type Description
---- ---- -----------
approle/ approle
n/a
token/ token
token based credentials
Create a
policy.
$ tee
sdc-approle-pol.hcl <<EOF
# Login with
AppRole
path
"auth/approle/login" {
capabilities = [ "create",
"read" ]
}
# Mount the
AppRole auth method
path
"sys/auth/approle" {
capabilities = [ "create",
"read", "update", "delete", "sudo" ]
}
# Configure
the AppRole auth method
path
"sys/auth/approle/*" {
capabilities = [ "create",
"read", "update", "delete" ]
}
# Create and
manage roles
path
"auth/approle/*" {
capabilities = [ "create",
"read", "update", "delete", "list" ]
}
# Write ACL
policies
path
"sys/policy/*" {
capabilities = [ "create",
"read", "update", "delete", "list" ]
}
# Write test
data
path
"secret/test/MySQL/app-sdc/*" {
capabilities = [ "create",
"read", "update", "delete", "list" ]
}
EOF
$ vault
policy write sdc-approle-pol sdc-approle-pol.hcl
$ vault
policy list
admin
default
sdc-approle-pol
root
Create a role
with policy attached.
$ vault write
auth/approle/role/app-sdc policies="sdc-approle-pol"
secret_id_ttl=120m token_ttl="60m" token_max_ttl="120m"
Read the
role.
$ vault read
auth/approle/role/app-sdc
Fetch RoleID
for app-sdc role.
$ vault read
auth/approle/role/app-sdc/role-id
Generate a new
SecretID for app-sdc role.
$ vault write
-f auth/approle/role/app-sdc/secret-id
$ vault read
auth/approle/role/app-sdc/role-id
Key Value
--- -----
role_id 36015ef7-875a-3765-4e74-e6b1ccdc5d3b
$ vault write
-f auth/approle/role/app-sdc/secret-id
Key Value
--- -----
secret_id
84d34edc-cb7f-0eb8-dc59-f91c47e9e9fe
secret_id_accessor a8f01a19-ec71-f70c-c564-0457e7a6213a
Authenticate with
Role ID and Secret ID to receive AppRole token.
$ vault write
auth/approle/login role_id="<role-id>" secret_id="<secret-id>"
Once the
authentication successful, Vault will provide a token to the application that
can be used to request secrets.
Streamsets
can authenticate with Vault using the Role ID in properties, and Secret ID provided
by a file.
Update credentialStore.vault.config.role.id=
in /etc/sdc/credential-stores.properties
Create /etc/sdc/vault-secret-id
file with <secret-id>
Using admin
user’s token, write a Secret to the path secret/test/MySQL/app-sdc
$ vault kv
put secret/test/MySQL/app-sdc sdc_mysql_user=<user> sdc_mysql_pass=<passwd>
Read secrets
using the AppRole token.
$ VAULT_TOKEN=<AppRole_token>
vault kv get secret/test/MySQL/app-sdc
Alternativtely,
you can first authenticate with vault using the client_token.
$ vault login <AppRole_token>
$ vault kv
get secret/test/MySQL/app-sdc
Use the
following notation for credentials in streamsets pipeline to access the secrets in Vault.
${vault:read("secret/test/MySQL/app-sdc/sdc_mysql_user","value")}
${vault:read("secret/test/MySQL/app-sdc/sdc_mysql_pass","value")}
MySQL Dynamic Credentials with Vault
Enable the
database secrets engine
$ vault
secrets enable -description="database dynamic secret storage" database
Configure
Vault with MySQL plugin and connection information by passing
role as read from any of the tables or update specific table and to be able to
read from any of the tables etc. A role is a logical name that maps to a policy
used to generate credentials.
$ vault write database/config/<mysql-name> \
plugin_name=mysql-database-plugin \
allowed_roles=<my-role> \
connection_url="{{username}}:{{password}}@tcp(127.0.0.1:3306)/"
\
username="root" password="mysql"
Configure allowed role to create dynamic credential. The values within the {{value}} will be filled in by
vault.
$ vault write
database/roles/<my-role> \
db_name=<mysql-name> \
creation_statements="CREATE USER
'{{name}}'@'%' IDENTIFIED BY '{{password}}';GRANT SELECT ON *.* TO
'{{name}}'@'%';" \
default_ttl="1h" max_ttl="24h"
Create a
policy. The client app needs to be able to read from the <my-role> role endpoint.
$ tee <my-role>-db-read.hcl <<EOF
path
"database/creds/<my-role>" {
capabilities
= ["read"]
}
EOF
$ vault
policy write <my-role>-db-read <my-role>-db-read.hcl
Create a new
token with <my-role>-db-read policy
$ vault token
create -policy="<my-role>-db-read" -wrap-ttl=20m
Generate a
new set of credentials. They do not exist until they are accessed.
$ vault read
database/creds/<my-role>
