0

I'm currently trying to get the spring cloud config server up and running as spring boot app. I have configured the dependencies in pom.xml for /actuator and also have helm release template with appliaction.properties as configmap as shown below. After deploying when I describe the pod I keep getting failing liveness/readiness probes because of which pod goes into endless loop of restarts. Any pointers to what I'm doing wrong would be great.

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.5.0</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>

    <groupId>com.asdf.asdf.configserver</groupId>
    <artifactId>asdf-config-server</artifactId>
    <version>0.0.1-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <java.version>1.8</java.version>
        <spring-cloud.version>2020.0.3</spring-cloud.version>
    </properties>

    <dependencies>
<!--        <dependency>-->
<!--            <groupId>org.springframework.cloud</groupId>-->
<!--            <artifactId>spring-cloud-starter-bus-amqp</artifactId>-->
<!--        </dependency>-->      
        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-config-server</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-actuator</artifactId>
        </dependency>
        <dependency>
            <groupId>io.micrometer</groupId>
            <artifactId>micrometer-core</artifactId>
        </dependency>
        <dependency>
            <groupId>io.micrometer</groupId>
            <artifactId>micrometer-registry-prometheus</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-security</artifactId>
        </dependency>
    </dependencies>
    <dependencyManagement>
        <dependencies>
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-dependencies</artifactId>
                <version>${spring-cloud.version}</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>
        </dependencies>
    </dependencyManagement>
<repositories>
        <repository>
            <id>maven-public</id>
            <url>https://nexus.tools.asdfasd.asdfasdf.com/repository/maven-public/</url>
        </repository>
        <repository>
            <id>releases</id>
            <url>https://nexus.tools.fxi-asdfasdf.asdf.com/repository/maven-releases/</url>
        </repository>
        <repository>
            <id>snapshots</id>
            <url>https://nexus.tools.asdfasdf.com/repository/maven-snapshots/</url>
        </repository>
    </repositories>


    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <version>2.3.5.RELEASE</version>
            </plugin>
        </plugins>
    </build>

</project>

replicaCount: 1
java_app: true
memory_limit: 1024

memory_percent: 20

resources:
  limits:
    cpu: "0.5"
    # memory: 256Mi
  requests:
    cpu: "0.1"
    # memory: 256Mi


container:
  ports:
    - name: http
      containerPort: 8080
      servicePort: 80
      protocol: TCP

ingress:
  internal:
    enabled: true
    port: http

healthChecks:
  livenessProbe:
    httpGet:
      path: /actuator/health
      port: http
    initialDelaySeconds: 90
    timeoutSeconds: 25
    periodSeconds: 20
    failureThreshold: 3
  readinessProbe:
    httpGet:
      path: /actuator/health
      port: http
    initialDelaySeconds: 90
    timeoutSeconds: 25
    periodSeconds: 20
    failureThreshold: 3


# Container environment variables
extraVars:
 - name: SPRING_CONFIG_LOCATION
   value: "/spring/config/"


configMaps:
  /spring/config:
    inline:
      - name: application.properties
        content: |-
          server.port=8080
          spring.application.name=asdf-configserver-service 

          spring.cloud.config.server.git.uri=https://github.asdfad.com/asdfa/configs
          spring.cloud.config.server.git.password={GITHUB_TOKEN}

          spring.cloud.config.server.git.default-label=test
          spring.cloud.config.server.git.searchPaths={application}
          
          management.metrics.tags.application=asdf-configserver-service
          management.metrics.tags.version=@project.version@
          management.security.enabled=false
          management.endpoint.info.enabled=true
          management.endpoint.health.enabled=true
          management.endpoint.shutdown.enabled=true
          management.endpoints.jmx.exposure.include=health,info
          management.endpoints.web.exposure.include=*

          endpoints.health.sensitive=false
          security.basic.enabled=false
          management.endpoint.health.show-details=always
          management.server.ssl.enabled=false

          management.endpoint.metrics.enabled=true
          management.endpoint.prometheus.enabled=true
          management.metrics.export.prometheus.enabled=true

          info.app.name=adsfasdf-configserver-service
          info.app.description=asdfasd asdfasdf config server.
          info.app.version=1.0.0

akv2k8s:
  GITHUB_TOKEN:
    vault: asdfasdfasdfasdfzcxva
    type: secret
    secretName: github-token
Avi
  • 1,453
  • 4
  • 18
  • 43
  • 2
    Have you tried to check if /actuator/health endpoint is working well? Disable temporarily healthChecks to avoid pod restarts and deploy the app. Then invoke this endpoint of your app and check if it is returning status 200 (and some response like "status": "UP"). This way you will be able to identify part which is failing - whether it's inability to access the endpoint by health checks or it's the app's configuration so the endpoint is not running correctly or something is wrong with the app so it doesn't return 200. Also check logs of your app. – Pawel Woroniecki Oct 05 '21 at 20:40
  • yeah thanks for the debugging tip. When I setup port-forward to actually see if the /actuator/health endpoint is accessible it doesn't load up "status": "UP" and I see app logs are fine but the error logs for the port-forward as below - **E1005 22:26:52.593027 56807 portforward.go:340] error creating error stream for port 9000 -> 8080: Timeout occured** – Avi Oct 05 '21 at 21:31
  • @Avi Can you try a solution from Spring documentation for [Kubernetes probes](https://docs.spring.io/spring-boot/docs/2.3.0.RELEASE/reference/html/production-ready-features.html#production-ready-kubernetes-probes) with different PATHs for liveness and readiness probes? path: /actuator/health/liveness path: /actuator/health/readiness – Andrew Skorkin Oct 06 '21 at 09:54
  • It shouldn't make any difference because also Spring documentation states clearly that while it provides separate endpoints for liveness and readiness probe (so it's worth to use it), it should still work if you use just /actuator/health. But generally that's right that it's worth to use these separate endpoints. @Avi, have you defined a service? I cannot see it in your code and in this case your container is inaccessible from outside. – Pawel Woroniecki Oct 06 '21 at 13:08
  • @PawelWoroniecki - yes you are right it doesn't make a difference if we append /liveness and /readiness to the /actuator/health although it's a good practice. Anyways I have finally resolved the /actuator/health endpoint issue and see the response below: – Avi Oct 06 '21 at 20:29
  • {"status":"DOWN","components":{"clientConfigServer":{"status":"UNKNOWN","details":{"error":"no property sources located"}},"configServer":{"status":"DOWN","details":{"repository":{"application":"app","profiles":"default"},"error":"org.springframework.cloud.config.server.environment.NoSuchRepositoryException: Cannot clone or checkout repository: https://github.sd.sd.sd.com/sdf/xyz-configs"}},"discoveryComposite":{"description":"Discovery Client not initialized","status":"UNKNOWN","components":{"discoveryClient":{"description":"Discovery Client not initialized","status":"UNKNOWN"}}}, – Avi Oct 06 '21 at 20:36
  • @Avi, Ok, so you should have these errors also in your application's logs. But now it's another topic because it's something related to your app's configuration and not to readiness/liveness probing so the best way would be to debug these logs and issues first. Anyway as you can see - app is not healthy so healthness probing indicated it correctly. In case you cannot resolve your issues, it would be better to good to try to reproduce it outside Kubernetes (or check if it occurs outside Kubernetes) and create new question focused on it. – Pawel Woroniecki Oct 07 '21 at 23:06
  • @PawelWoroniecki - Thanks for your response. You are right it's a different issue for which I have raised a new question -https://stackoverflow.com/questions/69481013/spring-cloud-config-server-git-connection-issues It would be good to understand why spring cloud config server fails to connect to a git repo which is a very basic functionality of cloud config server itself. – Avi Oct 07 '21 at 23:13

0 Answers0