Redis caching with Spring Boot

Hi,

A few example on how to handle Redis usage with Spring Boot. Also some examples on how to error handle exceptions and issues with Redis.

The code below will help you initialize your redis connect and how to use it. One thing to take notice is that redis keys are global so you must make sure that any method parameter you use with you keys and unique. For this reason below you have samples of custom key generators.

Redis Samples

 

Redis main configurations


import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.*;
import org.springframework.cache.interceptor.CacheErrorHandler;
import org.springframework.cache.interceptor.KeyGenerator;
import org.springframework.context.annotation.*;
import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
import org.springframework.data.redis.cache.RedisCacheManager;
import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;

import org.springframework.data.redis.serializer.StringRedisSerializer;
import org.springframework.util.StringUtils;

import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.Map;


@Configuration
@ComponentScan
@EnableCaching
@Profile({"dev","test"})
public class RedisCacheConfig extends CachingConfigurerSupport {
    @Override
    public CacheErrorHandler errorHandler() {

        return new CustomCacheErrorHandler();

    }

    protected final org.slf4j.Logger logger = LoggerFactory.getLogger(RedisCacheConfig.class);


    // This is a custom default keygenerator that is used if no other explicit key generator is specified
    @Bean
    public KeyGenerator keyGenerator() {
        return new KeyGenerator() {
            protected final org.slf4j.Logger logger = LoggerFactory.getLogger(RedisCacheConfig.class);

            @Override
            public Object generate(Object o, Method method, Object... objects) {
                return RedisCacheConfig.keyGeneratorProcessor(logger, o, method, null, objects);

            }
        };
    }

    // A custom key generator that generates a key based on the first method parameter while ignoring all other parameters
    @Bean("keyGeneratorFirstParamKey")
    public KeyGenerator keyGeneratorFirstParamKey() {

        return new KeyGenerator() {
            protected final org.slf4j.Logger logger = LoggerFactory.getLogger(RedisCacheConfig.class);

            @Override
            public Object generate(Object o, Method method, Object... objects) {

                return RedisCacheConfig.keyGeneratorProcessor(logger, o, method, 0, objects);
            }
        };
    }

    // A custom key generator that generates a key based on the second method parameter while ignoring all other parameters

    @Bean("keyGeneratorSecondParamKey")
    public KeyGenerator keyGeneratorSecondParamKey() {

        return new KeyGenerator() {
            protected final org.slf4j.Logger logger = LoggerFactory.getLogger(RedisCacheConfig.class);

            @Override
            public Object generate(Object o, Method method, Object... objects) {

                return RedisCacheConfig.keyGeneratorProcessor(logger, o, method, 1, objects);
            }
        };
    }

    // This is the main logic for creating cache keys
    public static String keyGeneratorProcessor(org.slf4j.Logger logger, Object o, Method method, Integer keyIndex, Object... objects) {

        // Retrieve all cache names for each anonation and compose a cache key prefix
        CachePut cachePutAnnotation = method.getAnnotation(CachePut.class);
        Cacheable cacheableAnnotation = method.getAnnotation(Cacheable.class);
        CacheEvict cacheEvictAnnotation = method.getAnnotation(CacheEvict.class);
        org.springframework.cache.annotation.CacheConfig cacheConfigClassAnnotation = o.getClass().getAnnotation(org.springframework.cache.annotation.CacheConfig.class);
        String keyPrefix = "";
        String[] cacheNames = null;

        if (cacheConfigClassAnnotation != null)
            cacheNames = cacheConfigClassAnnotation.cacheNames();


        if (cacheEvictAnnotation != null)
            if (cacheEvictAnnotation.value() != null)
                if (cacheEvictAnnotation.value().length > 0)
                    cacheNames = org.apache.commons.lang3.ArrayUtils.addAll(cacheNames, cacheEvictAnnotation.value());

        if (cachePutAnnotation != null)
            if (cachePutAnnotation.value() != null)
                if (cachePutAnnotation.value().length > 0)
                    cacheNames = org.apache.commons.lang3.ArrayUtils.addAll(cacheNames, cachePutAnnotation.value());

        if (cacheableAnnotation != null)
            if (cacheableAnnotation.value() != null)
                if (cacheableAnnotation.value().length > 0)
                    cacheNames = org.apache.commons.lang3.ArrayUtils.addAll(cacheNames, cacheableAnnotation.value());

        if (cacheNames != null)
            if (cacheNames.length > 0) {
                for (String cacheName : cacheNames)
                    keyPrefix += cacheName + "_";
            }

        StringBuilder sb = new StringBuilder();


        int parameterIndex = 0;
        for (Object obj : objects) {
            if (obj != null && !StringUtils.isEmpty(obj.toString())) {
                if (keyIndex == null)
                    sb.append(obj.toString());
                else if (parameterIndex == keyIndex) {
                    sb.append(obj.toString());
                    break;
                }
            }
            parameterIndex++;
        }


        String fullKey = keyPrefix + sb.toString();

        logger.debug("REDIS KEYGEN for CacheNames: " + keyPrefix + " with KEY: " + fullKey);

        return fullKey;
        //---------------------------------------------------------------------------------------------------------

        // Another example how to do custom cache keys
        // This will generate a unique key of the class name, the method name,
        // and all method parameters appended.
                /*StringBuilder sb = new StringBuilder();
                sb.append(o.getClass().getName());
                sb.append("-" + method.getName() );
                for (Object obj : objects) {
                    if(obj != null)
                        sb.append("-" + obj.toString());
                }

                if(logger.isDebugEnabled())
                    logger.debug("REDIS KEYGEN: " + sb.toString());
                return sb.toString();*/
    }

    // Create the redis connection here
    @Bean
    public JedisConnectionFactory jedisConnectionFactory() {
        JedisConnectionFactory jedisConFactory = new JedisConnectionFactory();

        jedisConFactory.setUseSsl(true);
        jedisConFactory.setHostName("127.0.0.1");
        jedisConFactory.setPort(6379);

        if (!StringUtils.isEmpty(mytoken)) {
            jedisConFactory.setPassword(mytoken);
        }

        jedisConFactory.setUsePool(true);
        jedisConFactory.afterPropertiesSet();

        return jedisConFactory;
    }

    @Bean
    public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer() {
        return new PropertySourcesPlaceholderConfigurer();
    }

    @Bean
    public RedisTemplate redisTemplate() {
        RedisTemplate redisTemplate = new RedisTemplate();
        redisTemplate.setConnectionFactory(jedisConnectionFactory());
        redisTemplate.setKeySerializer(new StringRedisSerializer());

        return redisTemplate;
    }

    // Cache configurations like how long data is cached
    @Bean
    public CacheManager cacheManager(RedisTemplate redisTemplate) {
        RedisCacheManager cacheManager = new RedisCacheManager(redisTemplate);

        Map cacheExpiration = new HashMap();


        cacheExpiration.put("USERS", 120);
        cacheExpiration.put("CARS", 3600):

        // Number of seconds before expiration. Defaults to unlimited (0)
        cacheManager.setDefaultExpiration(60);
        cacheManager.setExpires(cacheExpiration);
        return cacheManager;
    }
}

 

Redis Error/Exception Handling

 

public class CustomCacheErrorHandler implements CacheErrorHandler {


    protected final org.slf4j.Logger logger = LoggerFactory.getLogger(this.getClass());

    protected Gson gson = new GsonBuilder().create();


    @Override

    public void handleCacheGetError(RuntimeException exception,

                                    Cache cache, Object key) {

        logger.error("Error in REDIS GET operation for KEY: " + key, exception);
        try
        {
            if (cache.get(key) != null && logger.isDebugEnabled())
                logger.debug("Possible existing data which for the cache GET operation in REDIS Cache by KEY: " + key + " with TYPE: " + cache.get(key).get().getClass() + " and DATA: " + this.gson.toJson(cache.get(key).get()));
        } catch (Exception ex)
        {
            // NOTICE: This exception is not logged because this might occur because the cache connection is not established.
            // So if the initial exception that was thrown might have been the same, no connection to the cache server.
            // In such a case this is logged in above already, before the try catch.
        }
    }

    @Override

    public void handleCachePutError(RuntimeException exception, Cache cache,

                                    Object key, Object value) {

        logger.error("Error in REDIS PUT operation for KEY: " + key, exception);
        if(logger.isDebugEnabled())
            logger.debug("Error in REDIS PUT operation for KEY: " + key + " with TYPE: " + value.getClass() + " and DATA: " + this.gson.toJson(value), exception);
    }

    @Override

    public void handleCacheEvictError(RuntimeException exception, Cache cache,

                                      Object key) {

        logger.error("Error in REDIS EVICT operation for KEY: " + key, exception);
        try
        {
            if (cache.get(key) != null  && logger.isDebugEnabled())
                logger.debug("Possible existing data which for the cache EVICT operation in REDIS Cache by KEY: " + key + " with TYPE: " + cache.get(key).get().getClass() + " and DATA: " + this.gson.toJson(cache.get(key).get()));
        } catch (Exception ex)
        {
            // NOTICE: This exception is not logged because this might occur because the cache connection is not established.
            // So if the initial exception that was thrown might have been the same, no connection to the cache server.
            // In such a case this is logged in above already, before the try catch.
        }
    }

    @Override

    public void handleCacheClearError(RuntimeException exception,Cache cache){
        logger.error("Error in REDIS CLEAR operation ", exception);
    }

}

Custom Key Generator Example

 
@Cacheable(value = "USERS", keyGenerator = "keyGeneratorFirstParamKey")
    public UserData getUsers(String userId, Object data)
    {
        // Do something here
    }

Advertisements

Java + Spring Boot: Explicit Class Instances with Profiles, Beans and Qualifiers

Here is a simple example how to use beans to create instances of classes based on their profiles.

Firstly, you need is a base class or an interface that each class will inherit/implement.

The simple way of doing this is to just simply using the @Profile annotation on a class with the desired profile name. For example:

Use a dev profile for a class that is created when you dev profile is up and running and test profile when your test profile is used.

Then simply use the @Autowired annotation to on the base class/interface. The rest if induced automatically based on your profile. BUT this approach works fine in your classes and/or interface reside within the same package.

In a case that you are using a base class or an interface from another library/package and wanting to create a different class to be used with different profiles this might not work because you can’t change the used profile name in the base library/package.

In these cases you do the following:

  1. Create @Bean functions that return instantiate the desired class into an object based on a profile set to the bean function. The return value can be the base class or interface.
  2. Give the same bean name to all functions.
  3. On the @Autowired class member add the @Qualifier annotation giving the bean name which you want.

Spring will in this case induce the right object instance based on the profile in defined on a bean function.

@Bean(name="authenticationLogic")
    @Profile("dev")
    public BaseAuthentication getBaseAuth()
    {
        return new MockAuthenticationClient();
    }

    @Bean(name="authenticationLogic")
    @Autowired
    @Profile("test")
    public BaseAuthentication getMainAuth(MessageService messageService)
    {
        return new MainAuthClient(messageService);
    }



    @Autowired
    @Qualifier("authenticationLogic")
    private BaseAuthentication baseAuthentication;

Adding git information to your Spring Actuator Info endpoint with Gradle

Hi,

This is how you can add git related information in case you need that to keep track what functionality and code your development or test environments are using.

Configuration

First you need to add the following to your Gradle file:

plugins {
   id "com.gorylenko.gradle-git-properties" version "1.4.21"
}

apply plugin: 'com.gorylenko.gradle-git-properties'

After this you should have a new Gradle task that will generate a git.properties file that your Actuator Info Endpoint can use. This file by default is generated into the build path resources folder. So run this command before building your jar or docker image etc.

gradle generateGitProperties

Bonus

If you want to access the Info actutor enpoint to display that info from somewhere else you can do this:
@Autowired
InfoEndpoint infoEndpoint;

return new JSONObject(this.infoEndpoint.invoke()).toString();

Links

https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#howto-git-info

https://github.com/n0mer/gradle-git-properties

Helper Scripts for Docker, git and Java developers

Hi,

Here are some of my own scripts that I use when developing to ease my life:

Building a Java Gradle project, then building a docker image and pushing it

./gradlew test
if [ $? -eq 0 ]; then
    echo Tests OK
    gradle clean
    gradle generateGitProperties
    gradle bootRepackage
    ./cleandocker.sh
    docker rmi {your image name + tag}
    docker build -t {your image name + tag} .
    ./dockerregistrylogin.sh
    docker push {your image name + tag}
else
    echo Tests Failed
    exit 1
fi

Clean docker from all running containers and stopped ones

echo "Stoping all containers"
docker stop $(docker ps -a -q)
echo "Removing all containers"
docker rm $(docker ps -a -q)
echo "Starting dev environment"

Commit your code to git after gradle tests are successfull

./gradlew test
if [ $? -eq 0 ]; then
    echo Tests OK
    git add .
    git commit -m "$1"
    git push
else
    echo Tests Failed
    exit 1
fi

Merge your branch with your master

git checkout master
git pull origin master
git merge dev -m "$1"
git push origin master
git checkout dev

This one is for AWS Developers to run and get the AWS ECR docker login

#Notice: To use a certain profile for login define additional profiles like this: aws configure --profile awscli

function doAwsDockerRegistryLogin()
{
    local  myresult=$(aws ecr get-login --no-include-email --region eu-central-1 --profile awscli)
    echo "$myresult"
}

result=$(doAwsDockerRegistryLogin)   # or result=`myfunc`
eval $result

 

AWS ECS and Bitbucket Pipelines: Deploy your docker application

Hi,

Here are some tips and tricks on how to update an existing AWS ECS deployment.

NOTICE: This post assume that you have some knowledge on AWS, scripting, docker and Bitbucket.

The scripts and guide below will do the following:

  1. Clone external libraries needed for your project (Assumes that you have a multi-project application
  2. Build your application
  3. Build the docker image
  4. Push the docker image into Elastic Container Service own container registry
  5. Stop all tasks in a cluster to force the associated services to restart the tasks
    1. I am using this method of deploying to avoid making constant new task definitions. I find that unnecessary. My view is to have a deployment docker tag that your target service task definitions use. In this way you only make sure that you have one good task definition that you plan to you. If needed later update your task definition and service to use it. This deployment suggestion will not care of any other detail that except the docker image and cluster name.
  6. Then test your API or Web App in the cloud with some 3rd party tool in this case I am using Postman collection, Postman Environmental Settings and Newman

The above steps can be performed automatically when you make changes to a branch or manually from the commit view or branches view (on the row with the branch or commit id move you mouse on top of “…” to get the option “Run pipeline for a branch” and selected the manual pipeline option)

Needed steps:

  1. Create/have an IAM access keys for deployment into ECS and ECR from Bitbucket.
  2. Generate SSH keys in the Bitbucket repository where you plan to run your pipeline
  3. If you have any depended Bitbucket repositories copy the Public Key in Step 2 into that repository settings.
  4. Then in the primary repository from which you plan to deploy set environmental variables needed for the deployment.
  5. Create you pipeline with the example Bitbucket pipeline definition file and supplement scripts.

Step 1: AWS Access

You will need an access key/secrect to AWS with the following minimum policy settings:


{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "VisualEditor0",
            "Effect": "Allow",
            "Action": [
                "ecs:ListTasks",
                "ecr:CompleteLayerUpload",
                "ecr:GetAuthorizationToken",
                "ecs:StopTask",
                "ecr:UploadLayerPart",
                "ecr:InitiateLayerUpload",
                "ecr:BatchCheckLayerAvailability",
                "ecr:PutImage"
            ],
            "Resource": "*"
        }
    ]
}

Step 2: SSH Keys for Bitbucket

More info here on how to generate a key:

https://confluence.atlassian.com/bitbucket/use-ssh-keys-in-bitbucket-pipelines-847452940.html?_ga=2.23419699.803528104.1518767114-2088289616.1517306997

Notice: Remember to get the public key

Step 3: Other repositories (Optional)

From bitbucket:

If you want your Pipelines builds to be able to access a different Bitbucket repository (other than the repo where the builds run):

  1. Add an SSH key to the settings for the repo where the build will run, as described in Step 1 above (you can create a new key in Bitbucket Pipelines or use an existing key).
  2. Add the public key from that SSH key pair directly to settings for the other Bitbucket repo (i.e. the repo that your builds need to have access to).
    See Use access keys for details on how to add a public key to a Bitbucket repo.

Step 4: Setting up environmental variables

APPIMAGE_TESTENV_CLUSTER : The cluster name where to which the docker image is deployed to  in this case a test environment that is manually triggered

APPIMAGE_DEVENV_CLUSTER: A dev target cluster that is associated with the master branch and starts automatically

APPIMAGE_NAME: The docker image name (Notice: Must match the one in your service -> task definition)

APPIMAGE_TAG: the docker image tag The docker image name (Notice: Must match the one in your service -> task definition)

AWS_ACCESS_KEY_ID (SECURE)

AWS_SECRET_ACCESS_KEY (SECURE)

AWS_DEFAULT_REGION : The region where your cluster is located

REGISTRYNAME : The ECR registry name wherethe image is to be pushed

Step 5: Bitbucket Pipeline

Pipeline definitions

The sample pipeline script has two options:
* Custom/manual deployment in the custom section of the script
* Branches/automatic deployment in the branches section of the script

# This is a sample build configuration for Java (Gradle).
# Check our guides at https://confluence.atlassian.com/x/zd-5Mw for more examples.
# Only use spaces to indent your .yml configuration.
# -----
# You can specify a custom docker image from Docker Hub as your build environment.
image: atlassian/default-image:latest # Include Java support
options:
  max-time: 15 # 15 minutes incase something hangs up
  docker: true # Include Docker support
pipelines:
  custom: # Pipelines that can only be triggered manually
    test-env:
      - step:
          caches:
            - gradle
          script:
            # Modify the commands below to build your repository.
            # You must commit the Gradle wrapper to your repository
            # https://docs.gradle.org/current/userguide/gradle_wrapper.html
            - git clone {your external repository}
            - ls
            - bash ./scripts/bitbucket/buildproject.sh
            # Install AWS CLI and configure it
            #----------------------------------------
            - apt-get update
            - apt-get install jq
            - curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
            - unzip awscli-bundle.zip
            - ./awscli-bundle/install -b ~/bin/aws
            - export PATH=~/bin:$PATH
            #----------------------------------------
            - bash ./scripts/bitbucket/awsdev-dockerregistrylogin.sh
            - export IMAGE_NAME=$REGISTRYNAME/$APPIMAGE_NAME:$APPIMAGE_TAG
            - docker build -t $IMAGE_NAME .
            - docker push $IMAGE_NAME
            # This will stop all tasks in the AWS Cluster, by doing this the AWS Service will start the defined task definitions as new tasks.
            # NOTICE: This approach needs task definitions attached to services and no manually started tasks.
            - bash ./scripts/bitbucket/stopalltasks.sh $APPIMAGE_TESTENV_CLUSTER
            #----------------------------------------
            # Install Newman tool and test with postman collection and environmental settings your web app
            - npm install -g newman
            - ./scripts/newman-API-tests/run-testenv-tests.sh
            #----------------------------------------
    prod-env:
      - step:
          script:
            - echo "Manual triggers for deployments are awesome!"
  branches:
    master:
      - step:
          caches:
            - gradle
          script:
            #----------------------------------------
            # Modify the commands below to build your repository.
            # You must commit the Gradle wrapper to your repository
            # https://docs.gradle.org/current/userguide/gradle_wrapper.html
            - git clone {your external repository}
            - ls
            - bash ./scripts/bitbucket/buildproject.sh
            # Install AWS CLI and configure it
            #----------------------------------------
            - apt-get update
            - apt-get install jq
            - curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
            - unzip awscli-bundle.zip
            - ./awscli-bundle/install -b ~/bin/aws
            - export PATH=~/bin:$PATH
            #----------------------------------------
            # Build and install the newest docker image
            - bash ./scripts/bitbucket/awsdev-dockerregistrylogin.sh
            - export IMAGE_NAME=$REGISTRYNAME/$APPIMAGE_NAME:$APPIMAGE_TAG
            - docker build -t $IMAGE_NAME .
            - docker push $IMAGE_NAME
            #----------------------------------------
            # This will stop all tasks in the AWS Cluster, by doing this the AWS Service will start the defined task definitions as new tasks.
            # NOTICE: This approach needs task definitions attached to services and no manually started tasks.
            - bash ./scripts/bitbucket/stopalltasks.sh $APPIMAGE_DEVENV_CLUSTER
            #----------------------------------------
            # Install Newman tool and test with postman collection and environmental settings your web app
            - npm install -g newman
            - ./scripts/newman-API-tests/run-devenv-tests.sh
            #----------------------------------------

Stop all AWS tasks in the cloud

#!/usr/bin/env bash

echo "Getting tasks from AWS:"
echo "Cluster: $1 Service: $2"
#For a single task
#----------------
#task=$(aws ecs list-tasks --cluster "$1" --service-name "$2" | jq --raw-output '.taskArns[0] | split("/")[1]' )
 #echo "Stopping task: " $task

 #aws ecs stop-task --task $task --cluster "$1"
#----------------
tasks=$(aws ecs list-tasks --cluster "$1" --service-name "$2" | jq -r '.taskArns | map(.[0:]) | reduce .[] as $item (""; . + $item + " ")')
echo "Tasks received"
for task in $tasks; do
echo "Stopping task from AWS: " $task
	aws ecs stop-task --task $task --cluster "$1"
#echo "Task stopped."
done

Build your project

echo "Rebuilding project"
#gradlew_output=$(./gradlew build);
#echo "$gradlew_output"

./gradlew test
if [ $? -eq 0 ]; then
    echo Tests OK
    ./gradlew clean
    ./gradlew bootRepackage
else
    echo Tests Failed
fi

Get the AWS Login details for ECR docker login

#Notice: To use a certain profile for login define additional profiles like this: aws configure --profile awscli

function doAwsDockerRegistryLogin()
{
    local  myresult=$(aws ecr get-login --no-include-email)
    echo "$myresult"
}

result=$(doAwsDockerRegistryLogin)   # or result=`myfunc`
eval $result

Running API or WebApp tests with Newman and Postman

What do you need to make the tests work

  1. Create a new Postman collection
  2. Add your URLs to test
  3. Add scripts into the test tab
  4. When all your URLs in your collection are ready export them from the collection … button
  5. (Optional) Then create environment settings that you can export and use with newman

Bash script to run the newman tests

sleep 1m # Force a wait to make sure that all AWS services, your app, LBs etc are all loaded up and running

echo $DEVENV_URL
until $(curl --output /dev/null --silent --head --fail --insecure "$DEVENV_URL"); do
    printf '.'
    sleep 5
done

echo "Starting newman tests"
newman run {your postman collection}.postman_collection.json --environment "{your postman collection}.postman_environment.json" --insecure --delay-request 10<span id="mce_SELREST_start" style="overflow:hidden;line-height:0;"></span>

Postman scripts example

Retrieving a token from the body and inserting it into an environmental variable

var jsonData = JSON.parse(responseBody);

console.log("TOKEN:" + jsonData.token);

var str_array = jsonData.token.split('.');
for(var i = 0; i < str_array.length -1; i++) {
console.log("Array Item: " + i);
console.log(str_array[i])
console.log(CryptoJS.enc.Utf8.stringify(CryptoJS.enc.Base64.parse(str_array[i])));
}
postman.setEnvironmentVariable("token", jsonData.token);

Testing a response for success and body content

// example using pm.response.to.be*
pm.test("response must be valid and have a body", function () {
// assert that the status code is 200
pm.response.to.be.ok; // info, success, redirection, clientError, serverError, are other variants
// assert that the response has a valid JSON body
pm.response.to.be.withBody;
pm.response.to.be.json; // this assertion also checks if a body exists, so the above check is not needed
});

console.log("BODY:" + responseBody);

Links

http://2mins4code.com/2017/11/08/building-a-cicd-environment-with-bitbucket-pipelines-docker-aws-ecs-for-an-angular-app/

https://bitbucket.org/awslabs/amazon-ecs-bitbucket-pipelines-python/overview

https://confluence.atlassian.com/bitbucket/deploy-to-amazon-ecs-892623902.html

https://bitbucket-pipelines.prod.public.atl-paas.net/validator

https://confluence.atlassian.com/bitbucket/configure-bitbucket-pipelines-yml-792298910.html

https://confluence.atlassian.com/bitbucketserver/getting-started-with-bitbucket-server-and-aws-776640193.html

https://confluence.atlassian.com/bitbucket/java-with-bitbucket-pipelines-872013773.html

https://confluence.atlassian.com/bitbucket/use-ssh-keys-in-bitbucket-pipelines-847452940.html?_ga=2.23419699.803528104.1518767114-2088289616.1517306997

https://confluence.atlassian.com/bitbucket/environment-variables-794502608.html

https://docs.aws.amazon.com/cli/latest/reference/index.html#cli-aws

https://confluence.atlassian.com/bitbucket/run-pipelines-manually-861242583.html

https://www.npmjs.com/package/newman

http://blog.getpostman.com/2017/10/25/writing-tests-in-postman/

Using multiple AWS CLI profiles to manage development environments

To avoid setting global AWS credentials/access to the AWS CLI you can use CLI Profiles like this:

Create a new profile:
aws configure –profile {profilename}

Then use it by adding the profile after a command like in the example below:
aws ecr get-login –no-include-email –region eu-central-1 –profile {profilename}

 

Thats it, this will allows you to use different access keys and policies for different purposes without different AWS security configurations overriding others.

 

This is especially true when you want to test your code against real world security settings in the cloud that can’t have higher level rights.

Azure DocumentDB Code Samples: How to use Azure DocumentDB

Hi,

I’ve been working a bit on DocumentDB and thought of posting some sample code on how to use it. It might save people time and energy. I have had to work around some issues and headaches.

 

Notice, there is one thing you should take care of. Define your functions with the async keyword and use the await keyword on async function calls. Failure to do this will result in hanging application code.

Also, make sure that you are not accidentally calling synchronous functions from the Task class or some other place that is related to an async call. This will also hang the application code. Calling the Wait() function is one of the, also calling the Result property in the wrong place will result in the same problem.

A quote on the problem from a site:

“If you call the async method on the SAME thread that you then call Result or Wait() on, you will probably deadlock because once the async task has finished, it will wait to re-acquire the previous thread but it can’t because the thread is blocked on the call to Result/Wait()

you can use async tasks and await to avoid this problem but there is also another clever trick, certainly in newer versions of the .net framework and that is to invoke your async task on another thread, not on the one you are calling your method with. It is as simple as this:

var task = Task.Run(() => myMethodAsync());

which involves the method on a thread from the thread pool. When your calling thread then waits and blocks using Wait() or Result, the async task will NOT need to wait for your thread, it will re-acquire the one from the threadpool, finish and signal your waiting thread to allow it to continue!” http://lukieb.blogspot.fi/2016/07/calls-to-azure-documentdb-hang.html

 


/// <summary>
/// Sample class to be used for object serialization when handling data to the DocumentDB
/// </summary>
public class MyDocumentDBDataContainer
{
public String Title { get; set; }
public byte[] FileData { get; set; }

public String FileName { get; set; }

public class InnerDataContainer
{
public String Title { get; set; }
public int SomeNumber { get; set; }
}

public InnerDataContainer InnerData { get; set; }
}

public partial class Form1 : Form
{
/// <summary>
/// The DocumentDB address, end point where it exists
/// </summary>
private const string EndpointUrl = "https://mydocumentdbtest.documents.azure.com:443/";

/// <summary>
/// This can be the primary key you get from the Azure DocumentDB settings UI
/// </summary>
private const string AuthorizationKey =
"";

/// <summary>
/// A temp object for holding the documentDB database for processing
/// </summary>
private Database database;

/// <summary>
/// Same as above but for a collection
/// </summary>
private DocumentCollection collection;

public Form1()
{
InitializeComponent();
}

private async void button1_Click(object sender, EventArgs e)
{
Stream myStream = null;
OpenFileDialog openFileDialog1 = new OpenFileDialog();

openFileDialog1.InitialDirectory = "c:\\";
openFileDialog1.Filter = "txt files (*.txt)|*.txt|All files (*.*)|*.*";
openFileDialog1.FilterIndex = 2;
openFileDialog1.RestoreDirectory = true;

// Open a file to get some byte data to upload into DocumentDB
if (openFileDialog1.ShowDialog() == DialogResult.OK)
{
try
{
if ((myStream = openFileDialog1.OpenFile()) != null)
{
using (myStream)
{
try
{
MemoryStream ms = new MemoryStream();
myStream.CopyTo(ms);
await CreateDocumentClient(ms, openFileDialog1.FileName);
}
catch (Exception ex)
{
Exception baseException = ex.GetBaseException();
Console.WriteLine("Error: {0}, Message: {1}", ex.Message, baseException.Message);
}
}
}
}
catch (Exception ex)
{
MessageBox.Show("Error: Could not read file from disk. Original error: " + ex.Message);
}
}
}

/// <summary>
/// This is the main work horse here. The function will create a database, a collection and a sample document if they do not exist.
/// NOTICE: This is very important, define you functions with the async keyword and use the await keyword on async function calls. Failure to do this will result in haning application code.
/// Also make sure that you are not accidentally calling synchronous functions from the Task class or some other place that is related to a async call. This will also hang the application code.
/// More on this: http://lukieb.blogspot.fi/2016/07/calls-to-azure-documentdb-hang.html
/// Also notice that documentDB uses "links" to identify things. You will run into DocumentDB objects and a property SelfLink. This seems to just be a way of how things work.
/// </summary>
/// <param name="fileDataStream"></param>
/// <param name="fileName"></param>
/// <returns></returns>
private async Task CreateDocumentClient(MemoryStream fileDataStream, String fileName)
{
// Create a new instance of the DocumentClient
var client = new DocumentClient(new Uri(EndpointUrl), AuthorizationKey);

var databaseID = "myDBTest";
var collectionID = "myCollectionTest";

// Get the database and if it does not exist create it
this.database = this.GetDatabase(client, databaseID);
if (database == null)
{
this.database = await CreateDatabase(client, databaseID);
}

// Get the collection and if it does not exist then create it
this.collection = this.GetDocumentCollection(client, collectionID);
if(this.collection == null)
{
this.collection = await this.CreateCollection(client, collectionID);
}

// Create a temp data container, pass forward to be created in DocumentDB
MyDocumentDBDataContainer data = new MyDocumentDBDataContainer() { Title = fileName, InnerData = new MyDocumentDBDataContainer.InnerDataContainer() { Title = "InnerDataTitle", SomeNumber = 1 }, FileData = fileDataStream.ToArray(), FileName = fileName };
var result = await this.CreateDocument(client, data);

// Get the newly created document. Notice: In these code examples I use a title but you can use any identifier you wish.
var dataFromDocumentDB = this.ReadDocument(client, data.Title);

// Re-Create the file from the byte data from the DocumentDB storage
File.WriteAllBytes(dataFromDocumentDB.FileName, dataFromDocumentDB.FileData);
}



#region DocumentManagement

private async Task<Document> DeleteDocument(DocumentClient client, String documentTitle)
{
var documentToDelete =
client.CreateDocumentQuery<MyDocumentDBDataContainer>(this.collection.SelfLink)
.Where(e => e.Title == documentTitle)
.AsEnumerable()
.First();

Document doc = GetDocument(client, documentToDelete.Title);

var result = await client.DeleteDocumentAsync(doc.SelfLink);
return result.Resource;
}

private async Task<Document> UpdateDocument(DocumentClient client, String documentTitle)
{
// Update a Document

var singleDocument =
client.CreateDocumentQuery<MyDocumentDBDataContainer>(this.collection.SelfLink)
.Where(e => e.Title == documentTitle)
.AsEnumerable()
.First();

Document doc = GetDocument(client, singleDocument.Title);
MyDocumentDBDataContainer employeUpdated = singleDocument;
singleDocument.InnerData.SomeNumber = singleDocument.InnerData.SomeNumber + 1;
var result = await client.ReplaceDocumentAsync(doc.SelfLink, singleDocument);

return result.Resource;
}

private Document GetDocument(DocumentClient client, string id)
{
return client.CreateDocumentQuery(this.collection.SelfLink)
.Where(e => e.Id == id)
.AsEnumerable()
.First();
}

private MyDocumentDBDataContainer ReadDocument(DocumentClient client, String documentTitle)
{
// Read the collection

//var data = client.CreateDocumentQuery<MyDocumentDBDataContainer>(this.collection.SelfLink).AsEnumerable();
//foreach (var item in data)
//{
// Console.WriteLine(item.Title);
// Console.WriteLine(item.FileData);
// Console.WriteLine(item.InnerData.Title);
// Console.WriteLine("----------------------------------");
//}

// Read A Document - Where Name == "John Doe"
var singleDocument =
client.CreateDocumentQuery<MyDocumentDBDataContainer>(this.collection.SelfLink)
.Where(e => e.Title == documentTitle)
.AsEnumerable()
.FirstOrDefault();

return singleDocument;

//Console.WriteLine("-------- Read a document---------");
//Console.WriteLine(singleDocument.Title);
//Console.WriteLine(singleDocument.FileData);
//Console.WriteLine(singleDocument.InnerData.Title);
//Console.WriteLine("-------------------------------");
}

private async Task<Document> CreateDocument(DocumentClient client, object documentObject)
{

var result = await client.CreateDocumentAsync(collection.SelfLink, documentObject);
var document = result.Resource;

Console.WriteLine("Created new document: {0}\r\n{1}", document.Id, document);
return document;
}

#endregion

private async Task<Database> CreateDatabase(DocumentClient client, String databaseID)
{
Console.WriteLine();
Console.WriteLine("******** Create Database *******");

var databaseDefinition = new Database { Id = databaseID };
var result = await client.CreateDatabaseIfNotExistsAsync(databaseDefinition);
var database = result.Resource;

Console.WriteLine(" Database Id: {0}; Rid: {1}", database.Id, database.ResourceId);
Console.WriteLine("******** Database Created *******");

return database;
}

private DocumentCollection GetDocumentCollection(DocumentClient client, String collectionID)
{
var collections = client.CreateDocumentCollectionQuery(database.CollectionsLink,
"SELECT * FROM c WHERE c.id = '" + collectionID + "'").AsEnumerable();
if(collections.Count() > 0)
return collections.First();

return null;
}

private async Task QueryDocumentsWithPaging(DocumentClient client)
{
Console.WriteLine();
Console.WriteLine("**** Query Documents (paged results) ****");
Console.WriteLine();
Console.WriteLine("Quering for all documents");

var sql = "SELECT * FROM c";
var query = client.CreateDocumentQuery(collection.SelfLink, sql).AsDocumentQuery();

while (query.HasMoreResults)
{
var documents = await query.ExecuteNextAsync();

foreach (var document in documents)
{
Console.WriteLine(" Id: {0}; Name: {1};", document.id, document.name);
}
}

Console.WriteLine();
}

private Database GetDatabase(DocumentClient client, String databaseID)
{
//bool databaseExists = false;
Console.WriteLine();
Console.WriteLine();
Console.WriteLine("******** Get Databases List ********");

var databases = client.CreateDatabaseQuery().ToList();

foreach (var database in databases)
{
Console.WriteLine(" Database Id: {0}; Rid: {1}", database.Id, database.ResourceId);
return database;
}

Console.WriteLine();
Console.WriteLine("Total databases: {0}", databases.Count);

return null;
}

private async Task<DocumentCollection> CreateCollection(DocumentClient client, string collectionId, string offerType = "S1")
{

Console.WriteLine();
Console.WriteLine("**** Create Collection {0} in {1} ****", collectionId,
database.Id);

var collectionDefinition = new DocumentCollection { Id = collectionId };
var options = new RequestOptions { OfferType = offerType };
var result = await

client.CreateDocumentCollectionAsync(database.SelfLink,
collectionDefinition, options);
var collection = result.Resource;

Console.WriteLine("Created new collection");
//ViewCollection(collection);

return collection;
}

#region DifferentWaysOfDoingThings
private async Task<Document> CreateDocuments2(DocumentClient client, byte[] fileData)
{
Console.WriteLine();
Console.WriteLine("**** Create Documents ****");
Console.WriteLine();

dynamic document1Definition = new
{
name = "New Customer 1",
address = new
{
addressType = "Main Office",
addressLine1 = "123 Main Street",
location = new
{
city = "Brooklyn",
stateProvinceName = "New York"
},
postalCode = "11229",
countryRegionName = "United States"
},
fileDataBinary = fileData
};

Document document1 = CreateDocument2(client, document1Definition);
Console.WriteLine("Created document {0} from dynamic object", document1.Id);
Console.WriteLine();

return document1;
}

private async Task<Document> CreateDocument2(DocumentClient client, object documentObject)
{

var result = await client.CreateDocumentAsync(collection.SelfLink, documentObject);
var document = result.Resource;

Console.WriteLine("Created new document: {0}\r\n{1}", document.Id, document);
return document;
}

#endregion
}