Kubernetes 开发 SpringCloud (一)、使用SpringCloud Kubernetes组件进行服务发现

1Microsoft Windows [版本 10.0.17763.1]
2
3(c) 2018 Microsoft Corporation。 保留所有权利。
4
5c:\Users\mydlq>

1$ cd /users/mydlq

1$ mkdir .kube

 1<?xml version="1.0" encoding="UTF-8"?>
 2<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 3         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 4    <modelVersion>4.0.0</modelVersion>
 5
 6    <parent>
 7        <groupId>org.springframework.boot</groupId>
 8        <artifactId>spring-boot-starter-parent</artifactId>
 9        <version>2.1.6.RELEASE</version>
10        <relativePath/>
11    </parent>
12
13    <groupId>club.mydlq</groupId>
14    <artifactId>springcloud-k8s-discovery-demo</artifactId>
15    <version>0.0.1</version>
16    <name>springcloud-k8s-discovery-demo</name>
17    <description>SpringCloud Kubernetes Discovery Demo</description>
18
19    <properties>
20        <java.version>1.8</java.version>
21    </properties>
22
23    <dependencies>
24        <!--SpringBoot Web-->
25        <dependency>
26            <groupId>org.springframework.boot</groupId>
27            <artifactId>spring-boot-starter-web</artifactId>
28        </dependency>
29        <!--SpringBoot Actuator-->
30        <dependency>
31            <groupId>org.springframework.boot</groupId>
32            <artifactId>spring-boot-starter-actuator</artifactId>
33        </dependency>
34        <!--SpringCloud Kubernetes Discovery-->
35        <dependency>
36            <groupId>org.springframework.cloud</groupId>
37            <artifactId>spring-cloud-starter-kubernetes</artifactId>
38            <version>1.0.2.RELEASE</version>
39        </dependency>
40    </dependencies>
41
42    <build>
43        <plugins>
44            <plugin>
45                <groupId>org.springframework.boot</groupId>
46                <artifactId>spring-boot-maven-plugin</artifactId>
47            </plugin>
48        </plugins>
49    </build>
50
51</project>

 1import org.springframework.beans.factory.annotation.Autowired;
 2import org.springframework.cloud.client.discovery.DiscoveryClient;
 3import org.springframework.web.bind.annotation.*;
 4import java.util.List;
 5
 6@RestController
 7public class ServiceController {
 8
 9    @Autowired
10    private DiscoveryClient discoveryClient;
11
12    @GetMapping("/service")
13    public List<String> getServiceList(){
14        return discoveryClient.getServices();
15    }
16
17    @GetMapping("/instance")
18    public Object getInstance(@RequestParam("name") String name){
19        return discoveryClient.getInstances(name);
20    }
21
22}

 1spring:
 2  application:
 3    name: springcloud-kubernetes-discovery-demo
 4  cloud:
 5    kubernetes:
 6      discovery:
 7        enabled: true   #开启服务发现
 8
 9server:
10  port: 8080
11
12management:
13  server:
14    port: 8081
15  endpoints:
16    web:
17      exposure:
18        include: "*"

 1import org.springframework.boot.SpringApplication;
 2import org.springframework.boot.autoconfigure.SpringBootApplication;
 3import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
 4
 5@SpringBootApplication
 6@EnableDiscoveryClient
 7public class Application {
 8
 9    public static void main(String[] args) {
10        SpringApplication.run(Application.class, args);
11    }
12
13}

 1[
 2    "jenkins",
 3    "service-customer",
 4    "service-provider",
 5    "sonatype-nexus",
 6    "springboot-admin-k8s",
 7    "springboot-helloworld",
 8    "springboot-prometheus-demo",
 9    "springcloud-k8s-config",
10    "swagger-kubernetes"
11]

 1[
 2    {
 3        "instanceId": "9950be16-95d3-11e9-a040-000c29d98697",
 4        "serviceId": "springboot-helloworld",
 5        "secure": false,
 6        "metadata": {
 7            "app": "springboot-helloworld",
 8            "port.server": "8080",
 9            "admin": "enabled",
10            "group": "A",
11            "port.management": "8081"
12        },
13        "scheme": "http://",
14        "host": "10.20.2.89",
15        "port": 8081,
16        "uri": "http://10.20.2.89:8081"
17    }
18]

1spring:
2  cloud:
3    kubernetes:
4      discovery:
5        enabled: true

1spring:
2  cloud:
3    kubernetes:
4      discovery:
5        enabled: false

 1kind: Service
 2apiVersion: v1
 3metadata:
 4  name: service-A
 5  namespace: mydlqcloud
 6  labels:
 7    app: service-A
 8    group: A
 9spec:
10  type: ClusterIP
11  ports:
12    - name: server
13      port: 8080
14      targetPort: 8080
15  selector:
16    app: service-A

 1kind: Service
 2apiVersion: v1
 3metadata:
 4  name: service-B
 5  namespace: mydlqcloud
 6  labels:
 7    app: service-B
 8    group: B
 9spec:
10  type: ClusterIP
11  ports:
12    - name: server 
13      port: 8080
14      targetPort: 8080
15  selector:
16    app: service-B

1spring:
2  cloud:
3    kubernetes:
4      discovery:
5        enabled: true
6        serviceLabels:
7          group: A          #只发现带 group:A 的标签的服务

 1kind: Service
 2apiVersion: v1
 3metadata:
 4  name: service-A
 5  namespace: mydlqcloud
 6  labels:
 7    app: service-A
 8spec:
 9  type: ClusterIP
10  ports:
11    - name: server      #只设置一个端口
12      port: 8080
13      targetPort: 8080
14  selector:
15    app: service-A

 1kind: Service
 2apiVersion: v1
 3metadata:
 4  name: service-B
 5  namespace: mydlqcloud
 6  labels:
 7    app: service-B
 8spec:
 9  type: ClusterIP
10  ports:
11    - name: server      #端口1
12      port: 8080
13      targetPort: 8080
14    - name: management  #端口2
15      port: 8081
16      targetPort: 8081
17  selector:
18    app: service-B

1spring:
2  cloud:
3    kubernetes:
4      discovery:
5        enabled: true
6        primaryPortName: server     #Service A、Service B 都存在的端口名

1spring:
2  cloud:
3    kubernetes:
4      discovery:
5        enabled: true
6        primaryPortName: management  #Service B 存在的端口名

1spring:
2  cloud:
3    kubernetes:
4      discovery:
5        enabled: true
6        primaryPortName: none        #两个service都不存在的端口名

 1kind: Service
 2apiVersion: v1
 3metadata:
 4  name: service-A
 5  namespace: mydlqcloud
 6  labels:
 7    app: service-A
 8spec:
 9  type: ClusterIP
10  ports:
11    - name: server
12      port: 8080
13      targetPort: 8080
14  selector:
15    app: service-A

1spring:
2  cloud:
3    kubernetes:
4      discovery:
5        enabled: true
6        filter: "metadata.name != 'service-A'"

1spring:
2  cloud:
3    kubernetes:
4      discovery:
5        enabled: true
6        filter: "spec.ports.size() == 1 or spec.ports.size() == 2"

1spring:
2  cloud:
3    kubernetes:
4      discovery:
5        enabled: true
6        filter: "spec.type == 'NodePort'"

 1import io.fabric8.kubernetes.api.model.*;
 2import org.springframework.expression.ExpressionParser;
 3import org.springframework.expression.spel.standard.SpelExpressionParser;
 4import org.springframework.expression.spel.support.SimpleEvaluationContext;
 5import java.util.ArrayList;
 6import java.util.HashMap;
 7import java.util.List;
 8import java.util.Map;
 9
10/**
11 * 测试 SpringEL 表达式
12 */
13public class SpringEL {
14
15    public static void main(String[] args) {
16        // 输入待验证的表达式
17        String springEL = "metadata.name == 'test-project'";
18        // 创建模拟的 Service 对象
19        Service service = createService();
20        // 验证 SpringEL 表达式和对象中现有的值是否匹配
21        boolean isTrue = verification(service,springEL);
22        //输出结果
23        System.out.println(isTrue);
24    }
25
26    /**
27     * 创建模拟的 Service 对象，和 SpringCloud Kubernetes 服务发现实例保持一致
28     * @return
29     */
30    static Service createService(){
31        /** 设置Status **/
32        ServiceStatus serviceStatus = new ServiceStatus();
33        serviceStatus.setLoadBalancer(new LoadBalancerStatus());
34        /** 设置Spec **/
35        ServiceSpec serviceSpec = new ServiceSpec();
36        serviceSpec.setClusterIP("10.10.1.11");
37        // 设置type
38        serviceSpec.setType("NodePort");
39        // 设置 Selector
40        Map selector = new HashMap();
41        selector.put("app", "test-project");
42        serviceSpec.setSelector(selector);
43        // 设置端口1
44        ServicePort servicePort1 = new ServicePort();
45        servicePort1.setName("server");
46        servicePort1.setPort(8080);
47        servicePort1.setNodePort(30080);
48        servicePort1.setProtocol("TCP");
49        servicePort1.setTargetPort(new IntOrString("8080"));
50        // 设置端口2
51        ServicePort servicePort2 = new ServicePort();
52        servicePort2.setName("management  ");
53        servicePort2.setPort(8081);
54        servicePort2.setNodePort(30081);
55        servicePort2.setProtocol("TCP");
56        servicePort2.setTargetPort(new IntOrString("8081"));
57        // 将两个端口加如Spec
58        List<ServicePort> servicePortList = new ArrayList<>();
59        servicePortList.add(servicePort1);
60        servicePortList.add(servicePort2);
61        serviceSpec.setPorts(servicePortList);
62        /** 设置 Metadata **/
63        ObjectMeta objectMeta = new ObjectMeta();
64        objectMeta.setName("test-project");
65        objectMeta.setNamespace("mydlqcloud");
66        // 设置Label
67        Map labels = new HashMap();
68        labels.put("app", "test-project");
69        labels.put("group", "b");
70        objectMeta.setLabels(labels);
71        objectMeta.setResourceVersion("3373499");
72        objectMeta.setCreationTimestamp("2019-06-23T16:24:39Z");
73        /** 设置Service **/
74        Service service = new Service();
75        service.setKind("Service");
76        service.setApiVersion("v1");
77        service.setStatus(serviceStatus);
78        service.setSpec(serviceSpec);
79        service.setMetadata(objectMeta);
80        return service;
81    }
82
83    /**
84     * 使用 SpringEL 表达式测试 filter
85     * @param object
86     * @param springEL
87     * @return
88     */
89    static boolean verification(Object object,String springEL){
90        // 创建上下文环境
91        SimpleEvaluationContext evalCtxt = SimpleEvaluationContext.forReadOnlyDataBinding().withInstanceMethods().build();
92        // 创建SpEL表达式的解析器
93        ExpressionParser parser = new SpelExpressionParser();
94        // 验证表达式验证内容是否和对象中的值匹配并返回 Boolean 结果
95        return parser.parseExpression(springEL).getValue(evalCtxt, object, Boolean.class);
96    }
97
98}

  1public class KubernetesDiscoveryClient implements DiscoveryClient {
  2
  3    ......
  4
  5    /**
  6     * 获取描述信息，默认为下面固定的字符串
  7     * @return
  8     */
  9    @Override
 10    public String description() {
 11        return "Kubernetes Discovery Client";
 12    }
 13    
 14    /**
 15     * 获取服务实例列表
 16     * @param  serviceId K8S 中的 Service 名称
 17     * @return
 18     */
 19    @Override
 20    public List<ServiceInstance> getInstances(String serviceId) {
 21        Assert.notNull(serviceId,"[Assertion failed] - the object argument must not be null");
 22        // 根据服务名查找 K8S 中 endpoints 信息和 EndpointSubset 列表
 23        Endpoints endpoints = this.client.endpoints().withName(serviceId).get();
 24        List<EndpointSubset> subsets = getSubsetsFromEndpoints(endpoints);
 25        // 创建 ServiceInstance 列表，用于存储通过 API 获取过来的 Service 实例信息
 26        List<ServiceInstance> instances = new ArrayList<>();
 27        // 判断 Subsets 是否为空，为空则代表 Endpoints 没有关联的 Pod，相当于 Service 是没有关联 Pod 的无效服务
 28        if (!subsets.isEmpty()) {
 29            // 获取 Service 信息、Service Metadata 数据、和自定义参数对象 properties 中的 metadataProps 参数
 30            final Service service = this.client.services().withName(serviceId).get();
 31            final Map<String, String> serviceMetadata = new HashMap<>();
 32            KubernetesDiscoveryProperties.Metadata metadataProps = this.properties.getMetadata();
 33            // 根据配置文件中是否设置 isAddLabels 来决定是否将自定义 Annotations 加入到 serviceMetadata
 34            if (metadataProps.isAddLabels()) {
 35                Map<String, String> labelMetadata = getMapWithPrefixedKeys(
 36                        service.getMetadata().getLabels(),
 37                        metadataProps.getLabelsPrefix());
 38                if (log.isDebugEnabled()) {
 39                    log.debug("Adding label metadata: " + labelMetadata);
 40                }
 41                serviceMetadata.putAll(labelMetadata);
 42            }
 43            // 根据配置文件中是否设置 isAddAnnotations 来决定是否将自定义 Annotations 加入到 serviceMetadata
 44            if (metadataProps.isAddAnnotations()) {
 45                Map<String, String> annotationMetadata = getMapWithPrefixedKeys(
 46                        service.getMetadata().getAnnotations(),
 47                        metadataProps.getAnnotationsPrefix());
 48                if (log.isDebugEnabled()) {
 49                    log.debug("Adding annotation metadata: " + annotationMetadata);
 50                }
 51                serviceMetadata.putAll(annotationMetadata);
 52            }
 53            // 循环 EndpointSubset 获取部分参数拼凑 KubernetesServiceInstance 对象
 54            for (EndpointSubset s : subsets) {
 55                Map<String, String> endpointMetadata = new HashMap<>(serviceMetadata);
 56                // 检测参数是否设置 isAddPorts 参数，将自定义端口前缀加入到默认端口名前
 57                if (metadataProps.isAddPorts()) {
 58                    // 获取端口列表 Map
 59                    Map<String, String> ports = s.getPorts().stream()
 60                            .filter(port -> !StringUtils.isEmpty(port.getName()))
 61                            .collect(toMap(EndpointPort::getName,port -> Integer.toString(port.getPort())));
 62                    // 设置自定义前缀，设置 portMetadata Map 的 Key 改成（prefix + key）
 63                    Map<String, String> portMetadata = getMapWithPrefixedKeys(ports,metadataProps.getPortsPrefix());
 64                    if (log.isDebugEnabled()) {
 65                        log.debug("Adding port metadata: " + portMetadata);
 66                    }
 67                    endpointMetadata.putAll(portMetadata);
 68                }
 69                // 一个 Subset 中包含多个 Address 地址，从 EndpointSubset 中获取 addresses 列表
 70                List<EndpointAddress> addresses = s.getAddresses();
 71                // 根据 Address 循环获取 UID 和端口列表，然后拼接服务实例对象 KubernetesServiceInstance
 72                for (EndpointAddress endpointAddress : addresses) {
 73                    String instanceId = null;
 74                    if (endpointAddress.getTargetRef() != null) {
 75                        instanceId = endpointAddress.getTargetRef().getUid();
 76                    }
 77                    EndpointPort endpointPort = findEndpointPort(s);
 78                    // 通过上面获取的部分数据来拼接服务实例对象
 79                    instances.add(new KubernetesServiceInstance(instanceId, serviceId,
 80                            endpointAddress, endpointPort, endpointMetadata,
 81                            this.isServicePortSecureResolver
 82                                    .resolve(new DefaultIsServicePortSecureResolver.Input(
 83                                            endpointPort.getPort(),
 84                                            service.getMetadata().getName(),
 85                                            service.getMetadata().getLabels(),
 86                                            service.getMetadata().getAnnotations()))));
 87                }
 88            }
 89        }
 90        return instances;
 91    }
 92
 93    /**
 94     * 从 EndpointSubset 获取端口号列表
 95     * @param  s EndpointSubset 
 96     * @return
 97     */
 98    private EndpointPort findEndpointPort(EndpointSubset s) {
 99        // 获取端口号列表
100        List<EndpointPort> ports = s.getPorts();
101        EndpointPort endpointPort;
102        // 判断端口号数量，如果只有一个端口，就直接获取下标0的端口号
103        if (ports.size() == 1) {
104            endpointPort = ports.get(0);
105        }
106        else {
107            // 如果存在多个端口号，就以此根据 properties 中设置的 "PrimaryPortName" 参数，
108            // 根据这个参数值和 Service 端口名对比来过滤端口号。
109            Predicate<EndpointPort> portPredicate;
110            if (!StringUtils.isEmpty(properties.getPrimaryPortName())) {
111                portPredicate = port -> properties.getPrimaryPortName()
112                        .equalsIgnoreCase(port.getName());
113            }
114            // 如果为未设置筛选端口名称参数，则默认都为 true，都加入端口列表
115            else {
116                portPredicate = port -> true;
117            }
118            // 根据上面 boolean 值，来过滤端口号
119            endpointPort = ports.stream().filter(portPredicate).findAny()
120                    .orElseThrow(IllegalStateException::new);
121        }
122        return endpointPort;
123    }
124
125    /**
126     * 从 Endpoints 中获取 Subset
127     * @param  endpoints endpoints 对象
128     * @return 
129     */
130    private List<EndpointSubset> getSubsetsFromEndpoints(Endpoints endpoints) {
131        if (endpoints == null) {
132            return new ArrayList<>();
133        }
134        if (endpoints.getSubsets() == null) {
135            return new ArrayList<>();
136        }
137        return endpoints.getSubsets();
138    }
139
140    /**
141     * 如果自定义参数中设置了前缀，如 label前缀、port前缀、annotation前缀等，就将获取的示例信息对应的key前加上这个前缀
142     * @param  map    端口 map 集合
143     * @param  prefix 自定义端口名称前缀
144     * @return  
145     */
146    private Map<String, String> getMapWithPrefixedKeys(Map<String, String> map,String prefix) {
147        if (map == null) {
148            return new HashMap<>();
149        }
150        if (!StringUtils.hasText(prefix)) {
151            return map;
152        }
153        final Map<String, String> result = new HashMap<>();
154        map.forEach((k, v) -> result.put(prefix + k, v));
155        return result;
156    }
157
158    /**
159     * 获取经过 filter 中 springEL 表达式过滤后的服务列表
160     * @return 
161     */
162    @Override
163    public List<String> getServices() {
164        // 从参数配置类中获取 filter 中的 SprngEL 表达式
165        String spelExpression = this.properties.getFilter();
166        Predicate<Service> filteredServices;
167        // 如果未设置 filter 参数，则不过滤任何服务
168        if (spelExpression == null || spelExpression.isEmpty()) {
169            filteredServices = (Service instance) -> true;
170        }
171        // 如果设置 filter 参数，则根据 filter 中的 SpringEL 表达式进行验证，符合则为 true 否则为 false
172        else {
173            Expression filterExpr = this.parser.parseExpression(spelExpression);
174            filteredServices = (Service instance) -> {
175                Boolean include = filterExpr.getValue(this.evalCtxt, instance,
176                        Boolean.class);
177                if (include == null) {
178                    return false;
179                }
180                return include;
181            };
182        }
183        // 过滤服务，然后返回过滤后的服务列表
184        return getServices(filteredServices);
185    }
186
187    /**
188     * 根据 filter 过滤服务列表
189     * @param  filter 过滤列表
190     * @return      
191     */
192    public List<String> getServices(Predicate<Service> filter) {
193        return this.kubernetesClientServicesFunction.apply(this.client).list().getItems()
194                .stream().filter(filter).map(s -> s.getMetadata().getName())
195                .collect(Collectors.toList());
196    }
197
198}

  1@ConfigurationProperties("spring.cloud.kubernetes.discovery")
  2public class KubernetesDiscoveryProperties {
  3
  4    /** 设置为 true 则启动服务发现 */
  5    private boolean enabled = true;
  6
  7    /** 本地服务实例的名称。 */
  8    @Value("${spring.application.name:unknown}")
  9    private String serviceName = "unknown";
 10
 11    /**
 12     * 在从Kubernetes API服务器检索到服务之后过滤服务的SpEL表达式。
 13     */
 14    private String filter;
 15
 16    /** 设置使用HTTPS的安全端口。 */
 17    private Set<Integer> knownSecurePorts = new HashSet<Integer>() {
 18        {
 19            add(443);
 20            add(8443);
 21        }
 22    };
 23
 24    /**
 25     * 如果设置了，则只从Kubernetes API服务器获取与这些标签匹配的服务。
 26     */
 27    private Map<String, String> serviceLabels = new HashMap<>();
 28
 29    /**
 30     * 如果设置了该端口，则当为服务定义多个端口时，使用具有给定名称的端口作为主要端口。
 31     */
 32    private String primaryPortName;
 33
 34    private Metadata metadata = new Metadata();
 35
 36    public boolean isEnabled() {
 37        return this.enabled;
 38    }
 39
 40    public void setEnabled(boolean enabled) {
 41        this.enabled = enabled;
 42    }
 43
 44    public String getServiceName() {
 45        return this.serviceName;
 46    }
 47
 48    public void setServiceName(String serviceName) {
 49        this.serviceName = serviceName;
 50    }
 51
 52    public String getFilter() {
 53        return this.filter;
 54    }
 55
 56    public void setFilter(String filter) {
 57        this.filter = filter;
 58    }
 59
 60    public Set<Integer> getKnownSecurePorts() {
 61        return this.knownSecurePorts;
 62    }
 63
 64    public void setKnownSecurePorts(Set<Integer> knownSecurePorts) {
 65        this.knownSecurePorts = knownSecurePorts;
 66    }
 67
 68    public Map<String, String> getServiceLabels() {
 69        return this.serviceLabels;
 70    }
 71
 72    public void setServiceLabels(Map<String, String> serviceLabels) {
 73        this.serviceLabels = serviceLabels;
 74    }
 75
 76    public String getPrimaryPortName() {
 77        return primaryPortName;
 78    }
 79
 80    public void setPrimaryPortName(String primaryPortName) {
 81        this.primaryPortName = primaryPortName;
 82    }
 83
 84    public Metadata getMetadata() {
 85        return this.metadata;
 86    }
 87
 88    public void setMetadata(Metadata metadata) {
 89        this.metadata = metadata;
 90    }
 91
 92    @Override
 93    public String toString() {
 94        return new ToStringCreator(this).append("enabled", this.enabled)
 95                .append("serviceName", this.serviceName).append("filter", this.filter)
 96                .append("knownSecurePorts", this.knownSecurePorts)
 97                .append("serviceLabels", this.serviceLabels)
 98                .append("metadata", this.metadata).toString();
 99    }
100
101    /**
102     * Metadata 配置项
103     */
104    public class Metadata {
105
106        /**
107         * 设置好后，服务的Kubernetes标签将包含在返回的ServiceInstance的metadata中。
108         */
109        private boolean addLabels = true;
110
111        /**
112         * 当设置addtags时，这将用作metadata映射中键名的前缀。
113         */
114        private String labelsPrefix;
115
116        /**
117         * 设置好后，服务的Kubernetes注释将包含为返回的ServiceInstance的元数据。
118         */
119        private boolean addAnnotations = true;
120
121        /**
122         * 当设置addAnnotations时，它将用作metadata映射中键名的前缀。
123         */
124        private String annotationsPrefix;
125
126        /**
127         * 设置好后，任何已命名的Kubernetes服务端口都将包含ServiceInstance的metadata数据。
128         */
129        private boolean addPorts = true;
130
131        /**
132         * 当设置addPorts时，这将用作 metadata 映射中键名的前缀。
133         */
134        private String portsPrefix = "port.";
135
136        public boolean isAddLabels() {
137            return this.addLabels;
138        }
139        public void setAddLabels(boolean addLabels) {
140            this.addLabels = addLabels;
141        }
142        public String getLabelsPrefix() {
143            return this.labelsPrefix;
144        }
145        public void setLabelsPrefix(String labelsPrefix) {
146            this.labelsPrefix = labelsPrefix;
147        }
148        public boolean isAddAnnotations() {
149            return this.addAnnotations;
150        }
151        public void setAddAnnotations(boolean addAnnotations) {
152            this.addAnnotations = addAnnotations;
153        }
154        public String getAnnotationsPrefix() {
155            return this.annotationsPrefix;
156        }
157        public void setAnnotationsPrefix(String annotationsPrefix) {
158            this.annotationsPrefix = annotationsPrefix;
159        }
160        public boolean isAddPorts() {
161            return this.addPorts;
162        }
163        public void setAddPorts(boolean addPorts) {
164            this.addPorts = addPorts;
165        }
166        public String getPortsPrefix() {
167            return this.portsPrefix;
168        }
169        public void setPortsPrefix(String portsPrefix) {
170            this.portsPrefix = portsPrefix;
171        }
172
173        @Override
174        public String toString() {
175            return new ToStringCreator(this).append("addLabels", this.addLabels)
176                    .append("labelsPrefix", this.labelsPrefix)
177                    .append("addAnnotations", this.addAnnotations)
178                    .append("annotationsPrefix", this.annotationsPrefix)
179                    .append("addPorts", this.addPorts)
180                    .append("portsPrefix", this.portsPrefix).toString();
181        }
182    }
183}