Friday, November 28, 2014

VXLAN tenants subnets and "VLAN tags" on Juno


  In case of GRE or VXLAN tenants L2 networks the VLAN tags you see in the output of "ovs-vsctl show" and in output of "ovs-ofctl dump-flows br-tun" (mod_vlan_vid) are only locally significant. This VLAN tags are not really L2 tags added to the frames leaving on the physical interface. They are only used by openvswitch to separate traffic on the br-int, so the different tap interfaces corresponding to different neutron subnets do not see each other's traffic. As far as this tags are not 12-bits segments , number 4096 is not important.

 
**************************
ON CONTROLLER
**************************

[root@juno1 ~(keystone_admin)]# neutron net-list
+--------------------------------------+-----------------+-----------------------------------------------------+
| id                                   | name            | subnets                                             |
+--------------------------------------+-----------------+-----------------------------------------------------+
| 90b574e2-f51a-423e-aef9-c201f6f68b76 | kashyap_private | 5fe9e3cc-feee-4f51-bed8-f4891bd8aafe 40.0.0.0/24    |
| 65cbd354-daae-41bb-9d3c-e58b1062be19 | public          | 147d5ecd-fe39-489e-8901-3b20a2c50148 192.168.1.0/24 |
| 8b2de478-de3f-448e-8ec1-8f973a762daf | boris_network   | 4142d2c6-220c-4b47-8147-cf512f7a753b 15.0.0.0/24    |
| 3fdb2eb7-fff8-4633-824b-1da4c38ccbd5 | kashyap_network | 8bee3a8d-7fac-4d05-84e3-bbe52a601084 70.0.0.0/24    |
| 951715cc-2a19-470e-9987-25b3b7906756 | demo_network    | d512bb49-29f0-4fe2-886a-10d880cc83fc 10.0.0.0/24    |
+--------------------------------------+-----------------+-----------------------------------------------------+
[root@juno1 ~(keystone_admin)]# ip netns exec qdhcp-90b574e2-f51a-423e-aef9-c201f6f68b76 ifconfig
lo: flags=73  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10
        loop  txqueuelen 0  (Local Loopback)
        RX packets 1  bytes 576 (576.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 1  bytes 576 (576.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

tap5532b72d-8c: flags=4163  mtu 1500
        inet 40.0.0.11  netmask 255.255.255.0  broadcast 40.0.0.255
        inet6 fe80::f816:3eff:fe6e:91a7  prefixlen 64  scopeid 0x20
        ether fa:16:3e:6e:91:a7  txqueuelen 0  (Ethernet)
        RX packets 27  bytes 2982 (2.9 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 12  bytes 2112 (2.0 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[root@juno1 ~(keystone_admin)]# ovs-vsctl show
f2113bd0-c4ca-4c4b-af16-928ff03e53da
    Bridge br-int
        fail_mode: secure
        Port "tap37cfc1fc-09"
            tag: 1
            Interface "tap37cfc1fc-09"
                type: internal
        Port int-br-ex
            Interface int-br-ex
                type: patch
                options: {peer=phy-br-ex}
        Port "qr-7ff72517-f3"
            tag: 4
            Interface "qr-7ff72517-f3"
                type: internal
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap2d26c49a-37"
            tag: 5
            Interface "tap2d26c49a-37"
                type: internal
        Port "tap5532b72d-8c"
            tag: 4    <==== tag 4 on Controller Node
            Interface "tap5532b72d-8c"

                type: internal
        Port "qr-bd92408c-b4"
            tag: 5
            Interface "qr-bd92408c-b4"
                type: internal
        Port "qr-a494fcc8-e5"
            tag: 2
            Interface "qr-a494fcc8-e5"
                type: internal
        Port "qr-4162d98e-5c"
            tag: 1
            Interface "qr-4162d98e-5c"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "tap305cff35-34"
            tag: 2
            Interface "tap305cff35-34"
                type: internal
    Bridge br-ex
        Port "qg-88b1ef62-2c"
            Interface "qg-88b1ef62-2c"
                type: internal
        Port phy-br-ex
            Interface phy-br-ex
                type: patch
                options: {peer=int-br-ex}
        Port "qg-d3e929c6-ba"
            Interface "qg-d3e929c6-ba"
                type: internal
        Port "enp2s0"
            Interface "enp2s0"
        Port "qg-5c6dd032-a8"
            Interface "qg-5c6dd032-a8"
                type: internal
        Port br-ex
            Interface br-ex
                type: internal
        Port "qg-7b037650-10"
            Interface "qg-7b037650-10"
                type: internal
    Bridge br-tun
        Port "vxlan-c0a80089"
            Interface "vxlan-c0a80089"
                type: vxlan
                options: {df_default="true", in_key=flow, local_ip="192.168.0.127", out_key=flow, remote_ip="192.168.0.137"}
        Port br-tun
            Interface br-tun
                type: internal
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
    ovs_version: "2.1.3"


************************************************************************
So kashyap_private network corresponds tag 4 on Controller
************************************************************************
**********************
ON COMPUTE
**********************
[root@juno2 ~(keystone_kashyap)]# nova list
+--------------------------------------+------------+--------+------------+-------------+------------------------------------------+
| ID                                   | Name       | Status | Task State | Power State | Networks                                 |
+--------------------------------------+------------+--------+------------+-------------+------------------------------------------+
| 1359ac92-8092-47bc-b7d6-ee474b641355 | CirrOS321  | ACTIVE | -          | Running     | kashyap_private=40.0.0.12, 192.168.1.155 |
+--------------------------------------+------------+--------+------------+-------------+------------------------------------------+

VM is running on kashyap_private network (40.0.0.0/24)

[root@juno2 ~(keystone_kashyap)]# nova show 1359ac92-8092-47bc-b7d6-ee474b641355
+--------------------------------------+----------------------------------------------------------+
| Property                             | Value                                                    |
+--------------------------------------+----------------------------------------------------------+
| OS-DCF:diskConfig                    | AUTO                                                     |
| OS-EXT-AZ:availability_zone          | nova                                                     |
| OS-EXT-STS:power_state               | 1                                                        |
| OS-EXT-STS:task_state                | -                                                        |
| OS-EXT-STS:vm_state                  | active                                                   |
| OS-SRV-USG:launched_at               | 2014-11-28T15:21:59.000000                               |
| OS-SRV-USG:terminated_at             | -                                                        |
| accessIPv4                           |                                                          |
| accessIPv6                           |                                                          |
| config_drive                         |                                                          |
| created                              | 2014-11-28T15:21:48Z                                     |
| flavor                               | m1.tiny (1)                                              |
| hostId                               | 7339f21099a5d4d918b8e69302c1dab2d5f7af63babf36d02057177d |
| id                                   | 1359ac92-8092-47bc-b7d6-ee474b641355                     |
| image                                | cirros (6f7d1877-9b6b-4530-b868-9fe42a71bca9)            |
| kashyap_private network              | 40.0.0.12, 192.168.1.155                                 |
| key_name                             | oskey37                                                  |
| metadata                             | {}                                                       |
| name                                 | CirrOS321                                                |
| os-extended-volumes:volumes_attached | []                                                       |
| progress                             | 0                                                        |
| security_groups                      | default                                                  |
| status                               | ACTIVE                                                   |
| tenant_id                            | 2a8be6536a864dd0b73782fd0fb2faff                         |
| updated                              | 2014-11-28T15:21:59Z                                     |
| user_id                              | bce3ed8aaa97447b8edfe3cf734b0793                         |
+--------------------------------------+----------------------------------------------------------+

****************************
Identifying qvo* port
****************************

[root@juno2 ~(keystone_kashyap)]# brctl show

bridge name    bridge id        STP enabled    interfaces
qbr23b93632-0d        8000.1e955c2e4f2e    no        qvb23b93632-0d
                            tap23b93632-0d

qbr37f299e0-de        8000.1a32e0906384    no        qvb37f299e0-de
qbra0fa9687-39        8000.f6fd5e4399e7    no        qvba0fa9687-39
qbrff853471-05        8000.9ef67b07ba76    no        qvbff853471-05

[root@juno2 ~(keystone_admin)]# virsh dumpxml 1359ac92-8092-47bc-b7d6-ee474b641355 | grep 23b93632-0d
     
      <source bridge='qbr23b93632-0d'/>
      <target dev='tap23b93632-0d'/>


***********************************************************
Tracking veth-pair ( qvb23b93632-0d, qvo23b93632-0d )
***********************************************************
[root@juno2 ~(keystone_kashyap)]# ovs-vsctl

79e82e7f-9040-4789-b8c8-d7d397ec230b
    Bridge br-tun
        Port "vxlan-c0a8007f"
            Interface "vxlan-c0a8007f"
                type: vxlan
                options: {df_default="true", in_key=flow, local_ip="192.168.0.137", out_key=flow, remote_ip="192.168.0.127"}
        Port br-tun
            Interface br-tun
                type: internal
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
    Bridge br-ex
        Port phy-br-ex
            Interface phy-br-ex
                type: patch
                options: {peer=int-br-ex}
        Port br-ex
            Interface br-ex
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qvoff853471-05"
            tag: 2
            Interface "qvoff853471-05"
        Port "qvoa0fa9687-39"
            tag: 1
            Interface "qvoa0fa9687-39"
        Port "qvod282b303-3a"
            tag: 5
            Interface "qvod282b303-3a"
        Port int-br-ex
            Interface int-br-ex
                type: patch
                options: {peer=phy-br-ex}
        Port "qvo23b93632-0d"
            tag: 6  <=== tag 6 on Compute Node
            Interface "qvo23b93632-0d"

        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port br-int
            Interface br-int
                type: internal
        Port "qvo37f299e0-de"
            tag: 1
            Interface "qvo37f299e0-de"
    ovs_version: "2.1.3"

****************************************************************************
So kashyap_private network corresponds tag 6 ( Compute Node)
****************************************************************************