Merge pull request #164 from cloudfoundry-incubator/develop

bump changes in development branch

README.md

@@ -16,48 +16,6 @@ Install and start [BOSH-Deployment](https://github.com/cloudfoundry/bosh-deploym
 
 Install [CF-deployment](https://github.com/cloudfoundry/cf-deployment/blob/master/cf-deployment.yml)
 
-Create and upload release
-
-```sh
-git clone https://github.com/cloudfoundry-incubator/app-autoscaler-release
-cd app-autoscaler-release
-./scripts/update
-bosh create-release
-bosh -e YOUR_ENV upload-release
-```
-
-Deploy app-autoscaler
-
-```sh
-bosh -e YOUR_ENV -d app-autoscaler \
-     deploy templates/app-autoscaler-deployment.yml \
-     --vars-store=bosh-lite/deployments/vars/autoscaler-deployment-vars.yml \
-     -v system_domain=bosh-lite.com \
-     -v cf_admin_password=<cf admin password> \
-     -v skip_ssl_validation=true
-```
-To deploy app-autoscaler with density, use `app-autoscaler-deployment-fewer.yml`
-```sh
-bosh -e YOUR_ENV -d app-autoscaler \
-     deploy templates/app-autoscaler-deployment-fewer.yml \
-     --vars-store=bosh-lite/deployments/vars/autoscaler-deployment-vars.yml \
-     -v system_domain=bosh-lite.com \
-     -v cf_admin_password=<cf admin password> \
-     -v skip_ssl_validation=true
-```
-Alternatively you can use cf-deployment vars file to provide the cf_admin_password
-
-```sh
-bosh -e YOUR_ENV -d app-autoscaler \
-     deploy templates/app-autoscaler-deployment.yml \
-     --vars-store=bosh-lite/deployments/vars/autoscaler-deployment-vars.yml \
-     -v system_domain=bosh-lite.com \
-     -v skip_ssl_validation=true \
-     --vars-file=<path to cf deployment vars file>
-```
-
-#### Deploy autoscaler with `client_credentials` flow
-
 Install the UAA CLI, `uaac`.
 
 ```sh
@@ -82,21 +40,51 @@ Create a new autoscaler client
 ```sh
 uaac client add "autoscaler_client_id" \
     --authorized_grant_types "client_credentials" \
-    --authorities "cloud_controller.read,cloud_controller.admin" \
+    --authorities "cloud_controller.read,cloud_controller.admin,uaa.resource" \
     --secret "autoscaler_client_secret"
 ```
 
-Deploy autoscaler with the newly created autoscaler client
+Create and upload release
+
+```sh
+git clone https://github.com/cloudfoundry-incubator/app-autoscaler-release
+cd app-autoscaler-release
+./scripts/update
+bosh create-release
+bosh -e YOUR_ENV upload-release
+```
+
+Deploy app-autoscaler with the newly created autoscaler client
+
+```sh
+bosh -e YOUR_ENV -d app-autoscaler \
+     deploy templates/app-autoscaler-deployment.yml \
+     --vars-store=bosh-lite/deployments/vars/autoscaler-deployment-vars.yml \
+     -v system_domain=bosh-lite.com \
+     -v cf_client_id=autoscaler_client_id \
+     -v cf_client_secret=autoscaler_client_secret \
+     -v skip_ssl_validation=true
+```
+To deploy app-autoscaler with density, use `app-autoscaler-deployment-fewer.yml`
+```sh
+bosh -e YOUR_ENV -d app-autoscaler \
+     deploy templates/app-autoscaler-deployment-fewer.yml \
+     --vars-store=bosh-lite/deployments/vars/autoscaler-deployment-vars.yml \
+     -v system_domain=bosh-lite.com \
+     -v cf_client_id=autoscaler_client_id \
+     -v cf_client_secret=autoscaler_client_secret \
+     -v skip_ssl_validation=true
+
+```
+Alternatively you can use cf-deployment vars file to provide the cf_client_id and cf_client_secret
 
 ```sh
 bosh -e YOUR_ENV -d app-autoscaler \
      deploy templates/app-autoscaler-deployment.yml \
      --vars-store=bosh-lite/deployments/vars/autoscaler-deployment-vars.yml \
      -v system_domain=bosh-lite.com \
-     -v autoscaler_client_id=autoscaler_client_id \
-     -v autoscaler_client_secret=autoscaler_client_secret \
      -v skip_ssl_validation=true \
-     -o example/operation/client-credentials.yml
+     --vars-file=<path to cf deployment vars file>
 ```
 
 #### Deploy autoscaler with external postgres database
@@ -106,7 +94,8 @@ bosh -e YOUR_ENV -d app-autoscaler \
      deploy templates/app-autoscaler-deployment.yml \
      --vars-store=bosh-lite/deployments/vars/autoscaler-deployment-vars.yml \
      -v system_domain=bosh-lite.com \
-     -v cf_admin_password=<cf admin password> \
+     -v cf_client_id=autoscaler_client_id \
+     -v cf_client_secret=autoscaler_client_secret \
      -v skip_ssl_validation=true \
      -v database_host=<database_host> \
      -v database_port=<database_port> \
@@ -124,7 +113,8 @@ bosh -e YOUR_ENV -d app-autoscaler \
      --vars-store=bosh-lite/deployments/vars/autoscaler-deployment-vars.yml \
      -o example/operation/bosh-dns.yml \
      -v system_domain=bosh-lite.com \
-     -v cf_admin_password=<cf admin password> \
+     -v cf_client_id=autoscaler_client_id \
+     -v cf_client_secret=autoscaler_client_secret \
      -v skip_ssl_validation=true
 ```
 For density deployment
@@ -134,7 +124,8 @@ bosh -e YOUR_ENV -d app-autoscaler \
      --vars-store=bosh-lite/deployments/vars/autoscaler-deployment-vars.yml \
      -o example/operation/bosh-dns-fewer.yml \
      -v system_domain=bosh-lite.com \
-     -v cf_admin_password=<cf admin password> \
+     -v cf_client_id=autoscaler_client_id \
+     -v cf_client_secret=autoscaler_client_secret \
      -v skip_ssl_validation=true
 ```
 >** It's advised not to make skip_ssl_validation=true for non-development environment

example/operation/bosh-dns-fewer.yml

@@ -6,7 +6,8 @@
     url: https://bosh.io/d/github.com/cloudfoundry/bosh-dns-aliases-release?v=0.0.3
     version: '0.0.3'
     sha1: b0d0a0350ed87f1ded58b2ebb469acea0e026ccc
-
+- type: remove
+  path: /releases/name=consul
 #add bosh-dns-aliases add-on
 - type: replace
   path: /addons?/-
@@ -15,6 +16,7 @@
     include:
       stemcell:
       - os: ubuntu-trusty
+      - os: ubuntu-xenial
     jobs:
     - name: bosh-dns-aliases
       release: bosh-dns-aliases

example/operation/bosh-dns.yml

@@ -6,7 +6,9 @@
     url: https://bosh.io/d/github.com/cloudfoundry/bosh-dns-aliases-release?v=0.0.3
     version: '0.0.3'
     sha1: b0d0a0350ed87f1ded58b2ebb469acea0e026ccc
-
+- type: remove
+  path: /releases/name=consul
+
 #add bosh-dns-aliases add-on
 - type: replace
   path: /addons?/-
@@ -15,6 +17,7 @@
     include:
       stemcell:
       - os: ubuntu-trusty
+      - os: ubuntu-xenial
     jobs:
     - name: bosh-dns-aliases
       release: bosh-dns-aliases

jobs/apiserver/spec

@@ -61,6 +61,9 @@ properties:
     description: "PEM-encoded server certificate for public api"
   autoscaler.api_server.public_server_key:
     description: "PEM-encoded server key for public api"
+  autoscaler.api_server.http_client_timeout:
+    description: "Http client imeout for apiserver to communicate with other autoscaler components, in milliseconds"
+    default: 60000
 
   autoscaler.api_server.scheduler.host:
     description: "Host where scheduler is running"
@@ -165,6 +168,10 @@ properties:
 
   autoscaler.cf.api:
     description: "the Cloud Foundry API endpoint"
+  autoscaler.cf.client_id:
+    description: "the client id to log in cf"
+  autoscaler.cf.secret:
+    description: "the secret to log in cf"
   autoscaler.cf.skip_ssl_validation:
     description: "Flag to skip ssl validation to connect to CC and UAA"
     default: false

jobs/apiserver/templates/config.json.erb

@@ -24,8 +24,11 @@
     'publicPort' => p('autoscaler.api_server.publicPort'),
     'infoFilePath' => "/var/vcap/jobs/apiserver/bin/info.json",
     'cfApi' => p('autoscaler.cf.api'),
+    'cfClientId' => p('autoscaler.cf.client_id'),
+    'cfClientSecret' => p('autoscaler.cf.secret'),
     'skipSSLValidation' => p('autoscaler.cf.skip_ssl_validation'),
     'cacheTTL' => p('autoscaler.api_server.cache_ttl'),
+    'httpClientTimeout' => p('autoscaler.api_server.http_client_timeout'),
     'db' => {
        'maxConnections' => p('autoscaler.api_server.db_config.max_connections'),
        'minConnections' => p('autoscaler.api_server.db_config.min_connections'),

jobs/eventgenerator/spec

@@ -79,6 +79,9 @@ properties:
   autoscaler.eventgenerator.server.port:
     description: "the listening port of server"
     default: 6105
+  autoscaler.eventgenerator.http_client_timeout:
+    description: "Http client imeout for eventgenerator to communicate with other autoscaler components"
+    default: 60s
 
   autoscaler.eventgenerator.ca_cert:
     description: "PEM-encoded CA certificate"
@@ -170,4 +173,6 @@ properties:
     description: "Number of consecutive failure to trip the circuit down"
     default: 3
 
-
+  autoscaler.eventgenerator.health.port:
+    description: "the listening port of health endpoint"
+    default: 6204

jobs/eventgenerator/templates/eventgenerator.yml.erb

@@ -37,6 +37,9 @@ server:
 
 logging:
   level: <%= p("autoscaler.eventgenerator.logging.level") %>
+http_client_timeout: <%= p("autoscaler.eventgenerator.http_client_timeout") %>
+health:
+  port: <%= p("autoscaler.eventgenerator.health.port") %>
 
 db:
   policy_db:

jobs/metricscollector/spec

@@ -68,10 +68,6 @@ properties:
     description: "the Cloud Foundry API endpoint"
   autoscaler.cf.grant_type:
     description: "the grant type to log in cf" 
-  autoscaler.cf.username:
-    description: "the user name to log in cf"
-  autoscaler.cf.password:
-    description: "the password to log in cf"
   autoscaler.cf.client_id:
     description: "the client id to log in cf"
   autoscaler.cf.secret:
@@ -89,6 +85,9 @@ properties:
   autoscaler.metricscollector.server.port:
     description: "the listening port of server"
     default: 6103
+  autoscaler.metricscollector.http_client_timeout:
+    description: "Http client imeout for metricscollector to communicate with other autoscaler components"
+    default: 60s
   autoscaler.metricscollector.ca_cert:
     description: "PEM-encoded CA certificate"
   autoscaler.metricscollector.server_cert:
@@ -108,3 +107,6 @@ properties:
     description: "the method to collect metrics from loggregator"
     default: "streaming"
 
+  autoscaler.metricscollector.health.port:
+    description: "the listening port of health endpoint"
+    default: 6203

jobs/metricscollector/templates/metricscollector.yml.erb

@@ -28,8 +28,6 @@
 cf:
   api:  <%= p("autoscaler.cf.api") %>
   grant_type: <%= p("autoscaler.cf.grant_type") %>
-  username:  <%= p("autoscaler.cf.username") %>
-  password:  <%= p("autoscaler.cf.password") %>
   client_id: <%= p("autoscaler.cf.client_id") %>
   secret: <%= p("autoscaler.cf.secret") %>
   skip_ssl_validation: <%= p("autoscaler.cf.skip_ssl_validation") %>
@@ -45,6 +43,9 @@ server:
 
 logging:
   level: <%= p("autoscaler.metricscollector.logging.level") %>
+http_client_timeout: <%= p("autoscaler.metricscollector.http_client_timeout") %>
+health:
+  port: <%= p("autoscaler.metricscollector.health.port") %>
 
 db:
   policy_db:
@@ -61,4 +62,5 @@ db:
 collector:
   refresh_interval: <%= p("autoscaler.metricscollector.collector.refresh_interval") %>
   collect_interval: <%= p("autoscaler.metricscollector.collector.collect_interval") %>
-  collect_method: <%= p("autoscaler.metricscollector.collector.collect_method") %>
+  collect_method: <%= p("autoscaler.metricscollector.collector.collect_method") %>
+  save_interval: <%= p("autoscaler.metricscollector.collector.save_interval") %>

jobs/operator/spec

@@ -22,10 +22,6 @@ properties:
     description: "the Cloud Foundry API endpoint"
   autoscaler.cf.grant_type:
     description: "the grant type to log in cf" 
-  autoscaler.cf.username:
-    description: "the user name to log in cf"
-  autoscaler.cf.password:
-    description: "the password to log in cf"
   autoscaler.cf.client_id:
     description: "the client id to log in cf"
   autoscaler.cf.secret:
@@ -117,31 +113,33 @@ properties:
   autoscaler.operator.logging.level:
      description: "the leve of logging"
      default: "info"
-
+  autoscaler.operator.http_client_timeout:
+    description: "Http client imeout for operator to communicate with other autoscaler components"
+    default: 60s
   autoscaler.operator.app_sync_interval: 
     description: "the time interval to sync applications"
     default: 24h
 
   autoscaler.operator.instance_metrics_db.refresh_interval:
     description: "the time interval to prune the instancemetrics database"
     default: 24h
-  autoscaler.operator.instance_metrics_db.cutoff_days:
+  autoscaler.operator.instance_metrics_db.cutoff_duration:
     description: "the cutoff days when pruning instancemetrics database"
-    default: 30
+    default: 24h
 
   autoscaler.operator.app_metrics_db.refresh_interval:
     description: "the time interval to prune the appmetrics database"
     default: 24h
-  autoscaler.operator.app_metrics_db.cutoff_days:
+  autoscaler.operator.app_metrics_db.cutoff_duration:
     description: "the cutoff days when pruning appmetrics database"
-    default: 30
+    default: 48h
 
   autoscaler.operator.scaling_engine_db.refresh_interval:
     description: "the time interval to prune the scalingengine database"
     default: 24h
-  autoscaler.operator.scaling_engine_db.cutoff_days:
+  autoscaler.operator.scaling_engine_db.cutoff_duration:
     description: "the cutoff days when pruning scalingengine database"
-    default: 30
+    default: 240h
 
   autoscaler.operator.scaling_engine.host:
     description: "Host where the scaling engine is running"
@@ -224,5 +222,7 @@ properties:
   autoscaler.operator.enable_db_lock:
     description: "Enable database lock flag"
     default: false
-
-
+
+  autoscaler.operator.health.port:
+    description: "the listening port of health endpoint"
+    default: 6208