创建管理工作站

本页面介绍如何首次创建管理员工作站虚拟机 (VM)。本页面还会介绍如何配置工作站的 Docker 守护程序以使用代理。

另请参阅升级现有管理员工作站。

概览

您将使用 HashiCorp Terraform 0.11 版在 vSphere 中预配管理员工作站虚拟机，并使用管理员工作站安装 GKE On-Prem。下载管理员工作站开放虚拟化设备 (OVA) 文件，并使用 Terraform 将虚拟机部署到 vSphere。

准备工作

  1. 下载管理员工作站 OVA。
  2. 阅读管理员工作站概览。

部署管理员工作站 OVA

如需部署 OVA，请执行以下步骤：

  1. 保存以下 Terraform 配置之一，具体取决于您使用的是 DHCP 还是静态 IP。请务必同时保存 TF 和 TFVARS 文件。TF 文件是 Terraform HCL 配置，而 TFVARS 文件声明了要在配置中使用的变量：

    dhcp.tf

    #########################
####### VARIABLES #######
#########################

# vSphere username
variable "vsphere_user" { }
# vSphere password
variable "vsphere_password" { }
# vSphere server address
variable "vsphere_server" { }
# Install this public key in the created VM
variable "ssh_public_key_path" { default = "~/.ssh/vsphere_workstation.pub" }
# vSphere network to use for the VM
variable "network" { default = "VM Network"}
# Hostname for the VM
variable "vm_name" { default = "vsphere-workstation" }
# vSphere datacenter to create this VM in
variable "datacenter" { }
# vSphere datastore to create this VM in
variable "datastore" { }
# vSphere cluster to create this VM in
variable "cluster" { }
# vSphere resource pool to create this VM in
variable "resource_pool" { }
# Number of CPUs for this VM. Recommended minimum 4.
variable "num_cpus" { default = 4 }
# Memory in MB for this VM. Recommended minimum 8192.
variable "memory" { default = 8192 }
# The VM template to clone
variable "vm_template" { }
# Enable the provided Docker registry. If you use your own registry, set to "false"
variable "registry_enable" { default = "true" }
# Username to set for the Docker registry
variable "registry_username" { default = "gke" }
# Password to set for the Docker registry
variable "registry_password" { default = "password" }
# Optional DNS hostname for the registry's certificate
variable "registry_dns_hostname" { default = "" }

#########################
##### FOR UPGRADING #####
#########################
# Path on disk to the htpasswd file
variable "registry_htpasswd" { default = "" } // filepath
# Path on disk to the certificate for the Docker registry in the admin workstation
variable "registry_cert" { default = "" } // filepath
# Path on disk to the registry's CA
variable "registry_ca" { default = "" } // filepath
# Path on disk to the registry's private key
variable "registry_private_key" { default = "" } // filepath

##########################
##########################

provider "vsphere" {
  version        = "~> 1.5"
  user           = "${var.vsphere_user}"
  password       = "${var.vsphere_password}"
  vsphere_server = "${var.vsphere_server}"

  # if you have a self-signed cert
  allow_unverified_ssl = true
}

### vSphere Data ###

data "vsphere_datastore" "datastore" {
  name          = "${var.datastore}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "vsphere_datacenter" "dc" {
  name = "${var.datacenter}"
}

data "vsphere_compute_cluster" "cluster" {
  name          = "${var.cluster}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "vsphere_resource_pool" "pool" {
  name          = "${var.resource_pool}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "vsphere_network" "network" {
  name          = "${var.network}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "vsphere_virtual_machine" "template_from_ovf" {
  name          = "${var.vm_template}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "template_file" "dhcp_ip_config" {
  template = <<EOF
network:
  version: 2
  ethernets:
    ens192:
      dhcp4: true
EOF
}

data "template_file" "user_data" {
  template = <<EOF
#cloud-config
apt:
  primary:
    - arches: [default]
      uri: http://us-west1.gce.archive.ubuntu.com/ubuntu/
write_files:
  - path: /etc/netplan/99-dhcp.yaml
    permissions: '0644'
    encoding: base64
    content: |
      $${dhcp_ip_config}
runcmd:
  - netplan apply
  - /var/lib/gke/guest-startup.sh $${reg_enable} $${reg_username} $${reg_password} $${reg_dns_hostname} $${reg_htpasswd} $${reg_cert} $${reg_private_key} $${reg_ca}
EOF
  vars = {
    dhcp_ip_config = "${base64encode(data.template_file.dhcp_ip_config.rendered)}"

    reg_enable = "${var.registry_enable}"
    reg_username = "${var.registry_username}"
    reg_password = "${var.registry_password}"
    reg_dns_hostname = "${var.registry_dns_hostname}"

    reg_htpasswd = ""
    reg_cert = ""
    reg_private_key = ""
    reg_ca = ""

    #########################
    ##### FOR UPGRADING #####
    # reg_htpasswd = "${file(var.registry_htpasswd)}"
    # reg_cert = "${file(var.registry_cert)}"
    # reg_private_key = "${file(var.registry_private_key)}"
    # reg_ca = "${file(var.registry_ca)}"
    #########################
  }
}

### vSphere Resources ###

resource "vsphere_virtual_machine" "vm" {
  name             = "${var.vm_name}"
  resource_pool_id = "${data.vsphere_resource_pool.pool.id}"
  datastore_id     = "${data.vsphere_datastore.datastore.id}"
  num_cpus         = "${var.num_cpus}"
  memory           = "${var.memory}"
  guest_id         = "${data.vsphere_virtual_machine.template_from_ovf.guest_id}"
  enable_disk_uuid = "true"
  scsi_type = "${data.vsphere_virtual_machine.template_from_ovf.scsi_type}"
  network_interface {
    network_id   = "${data.vsphere_network.network.id}"
    adapter_type = "${data.vsphere_virtual_machine.template_from_ovf.network_interface_types[0]}"
  }

  wait_for_guest_net_timeout = 15

  nested_hv_enabled = false
  cpu_performance_counters_enabled = false

  disk {
    label            = "disk0"
    size             = "${max(50, data.vsphere_virtual_machine.template_from_ovf.disks.0.size)}"
    eagerly_scrub    = "${data.vsphere_virtual_machine.template_from_ovf.disks.0.eagerly_scrub}"
    thin_provisioned = "${data.vsphere_virtual_machine.template_from_ovf.disks.0.thin_provisioned}"
  }

  cdrom {
    client_device = true
  }

  vapp {
    properties = {
      hostname    = "${var.vm_name}"
      public-keys = "${file(var.ssh_public_key_path)}"
      user-data   = "${base64encode(data.template_file.user_data.rendered)}"
    }
  }

  clone {
    template_uuid = "${data.vsphere_virtual_machine.template_from_ovf.id}"
  }
}

output "ip_address" {
  value = "${vsphere_virtual_machine.vm.default_ip_address}"
}

    dhcp.tfvars

    vsphere_user = "administrator@vsphere.local"
vsphere_password = ""
# vSphere server IP or DNS name
vsphere_server = ""
ssh_public_key_path = "~/.ssh/vsphere_workstation.pub"

vm_name = "admin-workstation"

datastore = ""
datacenter = ""
cluster = ""
resource_pool = ""
network = "VM Network"

num_cpus = 4
memory = 8192
vm_template = "gke-on-prem-admin-appliance-vsphere-1.0.2-gke.3"

# docker registry credentials
registry_enable = "true"
registry_username = "gke"
registry_password = "password"
registry_dns_hostname = ""

# only needed when re-creating the workstation and you want to
# reuse registry certs. Each value is a filepath relative to this file.
# registry_htpasswd=""
# registry_cert=""
# registry_ca=""
# registry_private_key=""

    static-ip.tf

    #########################
####### VARIABLES #######
#########################

# vSphere username
variable "vsphere_user" { }
# vSphere password
variable "vsphere_password" { }
# vSphere server address
variable "vsphere_server" { }
# Install this public key in the created VM
variable "ssh_public_key_path" { default = "~/.ssh/vsphere_workstation.pub" }
# vSphere network to use for the VM
variable "network" { default = "VM Network"}
# Hostname for the VM
variable "vm_name" { default = "vsphere-workstation" }
# vSphere datacenter to create this VM in
variable "datacenter" { }
# vSphere datastore to create this VM in
variable "datastore" { }
# vSphere cluster to create this VM in
variable "cluster" { }
# vSphere resource pool to create this VM in
variable "resource_pool" { }
# Number of CPUs for this VM. Recommended minimum 4.
variable "num_cpus" { default = 4 }
# Memory in MB for this VM. Recommended minimum 8192.
variable "memory" { default = 8192 }
# The VM template to clone
variable "vm_template" { }
# The IP address to assign this this VM
variable "ipv4_address" { }
# Netmask prefix length
variable "ipv4_netmask_prefix_length" { }
# Default gateway to use
variable "ipv4_gateway" { }
# DNS resolvers to use
variable "dns_nameservers" { }
# Enable the provided Docker registry. If you use your own registry, set to "false"
variable "registry_enable" { default = "true" }
# Username to set for the Docker registry
variable "registry_username" { default = "gke" }
# Password to set for the Docker registry
variable "registry_password" { default = "password" }
# Optional DNS hostname for the registry's certificate
variable "registry_dns_hostname" { default = "" }

#########################
##### FOR UPGRADING #####
#########################
# Path on disk to the htpasswd file
variable "registry_htpasswd" { default = "" } // filepath
# Path on disk to the certificate for the Docker registry in the admin workstation
variable "registry_cert" { default = "" } // filepath
# Path on disk to the registry's CA
variable "registry_ca" { default = "" } // filepath
# Path on disk to the registry's private key
variable "registry_private_key" { default = "" } // filepath

##########################
##########################

provider "vsphere" {
  version        = "~> 1.5"
  user           = "${var.vsphere_user}"
  password       = "${var.vsphere_password}"
  vsphere_server = "${var.vsphere_server}"

  # if you have a self-signed cert
  allow_unverified_ssl = true
}

### vSphere Data ###

data "vsphere_datastore" "datastore" {
  name          = "${var.datastore}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "vsphere_datacenter" "dc" {
  name = "${var.datacenter}"
}

data "vsphere_compute_cluster" "cluster" {
  name          = "${var.cluster}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "vsphere_resource_pool" "pool" {
  name          = "${var.resource_pool}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "vsphere_network" "network" {
  name          = "${var.network}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

data "vsphere_virtual_machine" "template_from_ovf" {
  name          = "${var.vm_template}"
  datacenter_id = "${data.vsphere_datacenter.dc.id}"
}

##########################
### IF USING STATIC IP ###
##########################
data "template_file" "static_ip_config" {
  template = <<EOF
network:
  version: 2
  ethernets:
    ens192:
      dhcp4: no
      dhcp6: no
      addresses: ["${var.ipv4_address}/${var.ipv4_netmask_prefix_length}"]
      gateway4: ${var.ipv4_gateway}
      nameservers:
        addresses: [${var.dns_nameservers}]
EOF
}

data "template_file" "user_data" {
  template = <<EOF
#cloud-config
apt:
  primary:
    - arches: [default]
      uri: http://us-west1.gce.archive.ubuntu.com/ubuntu/
write_files:
  - path: /tmp/static-ip.yaml
    permissions: '0644'
    encoding: base64
    content: |
      $${static_ip_config}
runcmd:
  - /var/lib/gke/guest-startup.sh $${reg_enable} $${reg_username} $${reg_password} $${reg_dns_hostname} $${reg_htpasswd} $${reg_cert} $${reg_private_key} $${reg_ca}
EOF
  vars = {
    static_ip_config = "${base64encode(data.template_file.static_ip_config.rendered)}"

    reg_enable = "${var.registry_enable}"
    reg_username = "${var.registry_username}"
    reg_password = "${var.registry_password}"
    reg_dns_hostname = "${var.registry_dns_hostname}"

    reg_htpasswd = ""
    reg_cert = ""
    reg_private_key = ""
    reg_ca = ""

    ########################
    #### FOR UPGRADING #####
    # reg_htpasswd = "${file(var.registry_htpasswd)}"
    # reg_cert = "${file(var.registry_cert)}"
    # reg_private_key = "${file(var.registry_private_key)}"
    # reg_ca = "${file(var.registry_ca)}"
    ########################
  }
}
##########################
### IF USING STATIC IP ###
##########################

### vSphere Resources ###

resource "vsphere_virtual_machine" "vm" {
  name             = "${var.vm_name}"
  resource_pool_id = "${data.vsphere_resource_pool.pool.id}"
  datastore_id     = "${data.vsphere_datastore.datastore.id}"
  num_cpus         = "${var.num_cpus}"
  memory           = "${var.memory}"
  guest_id         = "${data.vsphere_virtual_machine.template_from_ovf.guest_id}"
  enable_disk_uuid = "true"
  scsi_type = "${data.vsphere_virtual_machine.template_from_ovf.scsi_type}"
  network_interface {
    network_id   = "${data.vsphere_network.network.id}"
    adapter_type = "${data.vsphere_virtual_machine.template_from_ovf.network_interface_types[0]}"
  }

  wait_for_guest_net_timeout = 15

  nested_hv_enabled = false
  cpu_performance_counters_enabled = false

  disk {
    label            = "disk0"
    size             = "${max(50, data.vsphere_virtual_machine.template_from_ovf.disks.0.size)}"
    eagerly_scrub    = "${data.vsphere_virtual_machine.template_from_ovf.disks.0.eagerly_scrub}"
    thin_provisioned = "${data.vsphere_virtual_machine.template_from_ovf.disks.0.thin_provisioned}"
  }

  cdrom {
    client_device = true
  }

  vapp {
    properties = {
      hostname    = "${var.vm_name}"
      public-keys = "${file(var.ssh_public_key_path)}"
      user-data   = "${base64encode(data.template_file.user_data.rendered)}"
    }
  }

  clone {
    template_uuid = "${data.vsphere_virtual_machine.template_from_ovf.id}"
  }
}

output "ip_address" {
  value = "${vsphere_virtual_machine.vm.default_ip_address}"
}

    static-ip.tfvars

    vsphere_user = "administrator@vsphere.local"
vsphere_password = ""
# vSphere server IP or DNS name
vsphere_server = ""
ssh_public_key_path = "~/.ssh/vsphere_workstation.pub"

vm_name = "admin-workstation"

datastore = ""
datacenter = ""
cluster = ""
resource_pool = ""
network = "VM Network"

num_cpus = 4
memory = 8192
vm_template = "gke-on-prem-admin-appliance-vsphere-1.0.2-gke.3"

# XXX.XXX.XXX.XXX
ipv4_address = "100.115.250.100"
# Eg 22
ipv4_netmask_prefix_length = "22"
# XXX.XXX.XXX.XXX
ipv4_gateway = "100.115.251.254"
# comma separated DNS servers
dns_nameservers = "8.8.8.8,8.8.4.4"

# docker registry credentials
registry_enable = "true"
registry_username = "gke"
registry_password = "password"
registry_dns_hostname = ""

# only needed when re-creating the workstation and you want to
# reuse registry certs. Each value is a filepath relative to this file.
# registry_htpasswd=""
# registry_cert=""
# registry_ca=""
# registry_private_key=""

  2. 创建 SSH 密钥值对，以便您可以向管理员工作站虚拟机进行身份验证：

    ssh-keygen -t rsa -f ~/.ssh/vsphere_workstation -N ""

  3. 为 Terraform 文件创建目录：

    mkdir [TERRAFORM_DIR]

    其中，[TERRAFORM_DIR] 是保存 Terraform 文件的目录路径。

  4. 将与您的主机网络配置匹配的 Terraform 配置复制到 [TERRAFORM_DIR]/terraform.tf 和 [TERRAFORM_DIR]/terraform.tfvars。

  5. 在 terraform.tfvars 中输入以下变量的值，这些变量指定您的 vSphere 环境的元素：

    • vsphere_server
    • vspher_user
    • vsphere_password
    • datastore
    • datacenter
    • cluster
    • resource_pool
    • network
    • vm_template（您在之前的步骤中通过运行 govc markastemplate 创建了这一变量。）

    输入 vm_name 的值。这是您要创建的管理员工作站的名称。它可以是您选择的任何名称。

    如果您使用的是静态 IP 地址，请输入以下变量的值：

    • ipv4_address
    • ipv4_netmask_prefix_length
    • ipv4_gateway
    • dns_nameservers

    如果您想使用在管理员工作站虚拟机上运行的 Docker Registry，请将 registry_enable 设置为“true”。同时输入这些变量的值：

    • registry_username
    • registry_password

    如果您想为管理员工作站虚拟机上运行的 Docker Registry 使用 DNS 主机名，请输入 registry_dns_hostname 的值。如果管理员工作站虚拟机没有静态 IP 地址，这将非常有用。

    如果要使用您自己的私有 Docker Registry，请将 registry_enable 设置为“false”。

  6. 创建管理员工作站虚拟机。这可能需要几分钟时间：

    cd [TERRAFORM_DIR]
terraform init && terraform apply -auto-approve -input=false

通过 SSH 连接到管理员工作站

  1. 切换到包含 Terraform 配置的目录。

  2. 检索管理员工作站虚拟机的 IP 地址：

    terraform output ip_address

  3. 使用生成的密钥和 IP 地址通过 SSH 连接到管理员工作站虚拟机：

    ssh -i ~/.ssh/vsphere_workstation ubuntu@$(terraform output ip_address)

    或者，如果您只想使用其地址：

    ssh -i ~/.ssh/vsphere_workstation ubuntu@[IP_ADDRESS]

验证管理员工作站是否已正确设置

验证是否已安装 gkectl 和 docker：

gkectl version
docker version

如果您将 registry_enable 设置为“true”，则可以验证 GKE On-Prem 组件的 Docker 映像是否已加载到管理员工作站虚拟机的 Docker 守护程序中：

docker images

加载所有 Docker 映像可能需要 15 分钟或更长时间。

最终，您可以验证管理员工作站虚拟机是否正在运行 Docker Registry：

docker ps

输出显示运行 Docker Registry 的容器：

IMAGE ...                      COMMAND ...              NAMES
docker-registry:v2.7.1-gke.0   "registry serve /e..."   registry2

配置 Docker 以通过代理拉取

如果您的网络是在代理后面运行，并且管理员工作站虚拟机运行 Docker Registry，则需要配置在管理员工作站上运行的 Docker 守护程序，以通过代理拉取映像：

  1. 收集 HTTP 和 HTTPS 代理的地址。

  2. 收集您需要联系但无需代理的每个主机的 IP 地址和主机名，包括：

    • vCenter 服务器的 IP 地址。
    • 所有 ESXi 主机的 IP 地址。
    • 您打算在负载平衡器上配置的 IP 地址。
    • 范围 192.168.0.0/16。

    在管理员工作站中，将以下地址添加到 no_proxy 变量中：

    printf -v no_proxy '%s,' [ADDRESSES];

    （可选）您可以将范围导出到一个环境变量以供稍后参考。请记住，应用和进程可能会使用此变量：

    export no_proxy="${no_proxy%,}";

  3. 打开 Docker 的配置文件，该文件存储在 /root/.docker/config.json、/home/[USER]/.docker/config.json 或其他位置，具体取决于您的设置。

  4. 在 config.json 中，添加以下行：

    "proxies": {

"default": {
       "httpProxy": "[HTTP_PROXY]",
       "httpsProxy": "[HTTPS_PROXY]",
       "noProxy": "[ADDRESSES]"
           }
}

    其中：

    • [HTTP_PROXY] 是您的 HTTP 代理（如果有）。
    • [HTTPS_PROXY] 是您的 HTTPS 代理（如果有）。
    • [ADDRESSES] 是用英文逗号分隔的列表，其中包含您需要在不使用代理的情况下访问的地址和主机名。

  5. 重启 Docker 以使更改生效：

    sudo systemctl restart docker

问题排查

如需了解详情，请参阅问题排查。

Terraform vSphere 提供商会话数限制

GKE On-Prem 使用 Terraform 的 vSphere 提供商在您的 vSphere 环境中启动虚拟机。提供商的会话数上限为 1000。当前实现不会在使用后关闭活动会话。如果您运行的会话过多，则可能会遇到 503 错误。

会话会在 300 秒后自动关闭。

表现

如果您运行的会话过多，则可能会遇到以下错误：

Error connecting to CIS REST endpoint: Login failed: body:
  {"type":"com.vmware.vapi.std.errors.service_unavailable","value":
  {"messages":[{"args":["1000","1000"],"default_message":"Sessions count is
  limited to 1000. Existing sessions are 1000.",
  "id":"com.vmware.vapi.endpoint.failedToLoginMaxSessionCountReached"}]}},
  status: 503 Service Unavailable
潜在原因

您的环境中运行的 Terraform 提供商会话过多。

解决方法

目前，此功能按预期运行。会话会在 300 秒后自动关闭。如需了解详情，请参阅 GitHub 问题 #618。

为 Docker 使用代理：oauth2: cannot fetch token

表现

使用代理时，您遇到以下错误：

oauth2: cannot fetch token: Post https://oauth2.googleapis.com/token: proxyconnect tcp: tls: oversized record received with length 20527
潜在原因

您提供的可能是 HTTPS 代理，而不是 HTTP。

解决方法

在 Docker 配置中，将代理地址更改为 http:// 而不是 https://。

验证许可是否有效

请务必验证您的许可是否有效，特别是在使用试用许可的情况下。如果您的 F5、ESXi 主机或 vCenter 许可已过期，您可能会遇到意外故障。

openssl 无法验证管理员工作站 OVA

表现

对管理员工作站 OVA 文件运行 openssl dgst 不会返回 Verified OK

潜在原因

OVA 文件存在问题，导致无法成功验证。

解决方法

按照下载管理员工作站 OVA 中的说明，尝试重新下载并部署管理员工作站 OVA。如果问题仍然存在，请与 Google 联系以获取帮助。

后续步骤

在安装 GKE On-Prem 之前，可以考虑以下几个可选的配置选项：

上一页
开始使用