자동 확장 정책이 있는 Dataproc 클러스터를 만듭니다.
코드 샘플
Java
이 샘플을 사용해 보기 전에 클라이언트 라이브러리 사용한 Dataproc 빠른 시작의 Java 설정 안내를 따르세요. 자세한 내용은 Dataproc Java API 참고 문서를 참조하세요.
Dataproc에 인증하려면 애플리케이션 기본 사용자 인증 정보를 설정합니다. 자세한 내용은 로컬 개발 환경의 인증 설정을 참조하세요.
import com.google.api.gax.longrunning.OperationFuture;
import com.google.cloud.dataproc.v1.AutoscalingConfig;
import com.google.cloud.dataproc.v1.AutoscalingPolicy;
import com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient;
import com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings;
import com.google.cloud.dataproc.v1.BasicAutoscalingAlgorithm;
import com.google.cloud.dataproc.v1.BasicYarnAutoscalingConfig;
import com.google.cloud.dataproc.v1.Cluster;
import com.google.cloud.dataproc.v1.ClusterConfig;
import com.google.cloud.dataproc.v1.ClusterControllerClient;
import com.google.cloud.dataproc.v1.ClusterControllerSettings;
import com.google.cloud.dataproc.v1.ClusterOperationMetadata;
import com.google.cloud.dataproc.v1.InstanceGroupAutoscalingPolicyConfig;
import com.google.cloud.dataproc.v1.InstanceGroupConfig;
import com.google.cloud.dataproc.v1.RegionName;
import com.google.protobuf.Duration;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
public class CreateClusterWithAutoscaling {
public static void createClusterwithAutoscaling() throws IOException, InterruptedException {
// TODO(developer): Replace these variables before running the sample.
String projectId = "your-project-id";
String region = "your-project-region";
String clusterName = "your-cluster-name";
String autoscalingPolicyName = "your-autoscaling-policy";
createClusterwithAutoscaling(projectId, region, clusterName, autoscalingPolicyName);
}
public static void createClusterwithAutoscaling(
String projectId, String region, String clusterName, String autoscalingPolicyName)
throws IOException, InterruptedException {
String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region);
// Configure the settings for the cluster controller client.
ClusterControllerSettings clusterControllerSettings =
ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
// Configure the settings for the autoscaling policy service client.
AutoscalingPolicyServiceSettings autoscalingPolicyServiceSettings =
AutoscalingPolicyServiceSettings.newBuilder().setEndpoint(myEndpoint).build();
// Create a cluster controller client and an autoscaling controller client with the configured
// settings. The clients only need to be created once and can be reused for multiple requests.
// Using a
// try-with-resources closes the client, but this can also be done manually with the .close()
// method.
try (ClusterControllerClient clusterControllerClient =
ClusterControllerClient.create(clusterControllerSettings);
AutoscalingPolicyServiceClient autoscalingPolicyServiceClient =
AutoscalingPolicyServiceClient.create(autoscalingPolicyServiceSettings)) {
// Create the Autoscaling policy.
InstanceGroupAutoscalingPolicyConfig workerInstanceGroupAutoscalingPolicyConfig =
InstanceGroupAutoscalingPolicyConfig.newBuilder()
.setMinInstances(2)
.setMaxInstances(100)
.setWeight(1)
.build();
InstanceGroupAutoscalingPolicyConfig secondaryWorkerInstanceGroupAutoscalingPolicyConfig =
InstanceGroupAutoscalingPolicyConfig.newBuilder()
.setMinInstances(0)
.setMaxInstances(100)
.setWeight(1)
.build();
BasicYarnAutoscalingConfig basicYarnApplicationConfig =
BasicYarnAutoscalingConfig.newBuilder()
.setScaleUpFactor(0.05)
.setScaleDownFactor(1.0)
.setScaleUpMinWorkerFraction(0.0)
.setScaleUpMinWorkerFraction(0.0)
.setGracefulDecommissionTimeout(Duration.newBuilder().setSeconds(3600).build())
.build();
BasicAutoscalingAlgorithm basicAutoscalingAlgorithm =
BasicAutoscalingAlgorithm.newBuilder()
.setCooldownPeriod(Duration.newBuilder().setSeconds(240).build())
.setYarnConfig(basicYarnApplicationConfig)
.build();
AutoscalingPolicy autoscalingPolicy =
AutoscalingPolicy.newBuilder()
.setId(autoscalingPolicyName)
.setWorkerConfig(workerInstanceGroupAutoscalingPolicyConfig)
.setSecondaryWorkerConfig(secondaryWorkerInstanceGroupAutoscalingPolicyConfig)
.setBasicAlgorithm(basicAutoscalingAlgorithm)
.build();
RegionName parent = RegionName.of(projectId, region);
// Policy is uploaded here.
autoscalingPolicyServiceClient.createAutoscalingPolicy(parent, autoscalingPolicy);
// Now the policy can be referenced when creating a cluster.
String autoscalingPolicyUri =
String.format(
"projects/%s/locations/%s/autoscalingPolicies/%s",
projectId, region, autoscalingPolicyName);
AutoscalingConfig autoscalingConfig =
AutoscalingConfig.newBuilder().setPolicyUri(autoscalingPolicyUri).build();
// Configure the settings for our cluster.
InstanceGroupConfig masterConfig =
InstanceGroupConfig.newBuilder()
.setMachineTypeUri("n1-standard-2")
.setNumInstances(1)
.build();
InstanceGroupConfig workerConfig =
InstanceGroupConfig.newBuilder()
.setMachineTypeUri("n1-standard-2")
.setNumInstances(2)
.build();
ClusterConfig clusterConfig =
ClusterConfig.newBuilder()
.setMasterConfig(masterConfig)
.setWorkerConfig(workerConfig)
.setAutoscalingConfig(autoscalingConfig)
.build();
// Create the cluster object with the desired cluster config.
Cluster cluster =
Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build();
// Create the Dataproc cluster.
OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest =
clusterControllerClient.createClusterAsync(projectId, region, cluster);
Cluster response = createClusterAsyncRequest.get();
// Print out a success message.
System.out.printf("Cluster created successfully: %s", response.getClusterName());
} catch (ExecutionException e) {
// If cluster creation does not complete successfully, print the error message.
System.err.println(String.format("createClusterWithAutoscaling: %s ", e.getMessage()));
}
}
}다음 단계
다른 Google Cloud 제품의 코드 샘플을 검색하고 필터링하려면 Google Cloud 샘플 브라우저를 참조하세요.