Since there is not Native support for IPSec in OpenVZ kernel, it is not possible to use openSwan, strongSwan or Racoon for IPSec VPN on OpenVZ VM. This has been changed, a plugin called kernel-libipsec was introduced after strongSwan Version 5.10. We can use this plugin with TUN to simulate a IPSec support in userspace.

First We have to check whether TUN/TAP is enabled for the VPS:

cat /dev/net/tun

We should get the output:

cat: /dev/net/tun: File descriptor in bad state

Otherwise we cannot continue since TUN/TAP support is a prerequisite.

Using APT.

apt-get install strongswan libcharon-extra-plugins

Download the source code and decompress.

wget http://download.strongswan.org/strongswan.tar.gz && tar zxvf strongswan* && cd strongswan*

Install the packaged which is needed for the compiling:

apt-get update && apt-get install build-essential libgmp3-dev openssl libssl-dev

If you want to integrate StrongSwan with Radius:

./configure --sysconfdir=/etc --disable-sql --disable-mysql --disable-ldap --enable-dhcp --enable-eap-identity --enable-eap-mschapv2 --enable-md4 --enable-xauth-eap --enable-eap-peap --enable-eap-md5 --enable-openssl --enable-shared --enable-unity --enable-eap-tls --enable-eap-ttls --enable-eap-tnc --enable-eap-dynamic --enable-addrblock --enable-eap-radius --enable-radattr --enable-nat-transport --enable-cisco-quirks --enable-kernel-netlink --enable-kernel-libipsec

Otherwise you can use for a minimal installation:

./configure --sysconfdir=/etc --disable-sql --disable-mysql --disable-ldap --enable-openssl --enable-shared --enable-kernel-libipsec

 make && make install

Edit strongSwan IPsec configuration file:

vi /etc/ipsec.conf

/etc/ipsec.conf:

# ipsec.conf - strongSwan IPsec configuration file 
config setup
    uniqueids=never
conn %default
    rekey=no
    left=%defaultroute
    leftsubnet=0.0.0.0/0
    right=%any
    dpddelay=300s
    dpdtimeout=1h
    dpdaction=clear
    auto=add
conn ikecommon
    rightsourceip=10.11.0.0/24
    #modeconfig=push
    #esp=aes128-sha1-modp2048
    compress=yes
conn ikev1pskaggressive
    aggressive=yes
    #type=tunnel
    also=ikev1psk
conn ikev1psk
    #authby=xauthpsk
    leftauth=psk
    rightauth=psk
    rightauth2=xauth
    #xauth=server
    also=ikev1
conn ikev1
    keyexchange=ikev1
    #ikelifetime=60m
    #keylife=20m
    #rekeymargin=3m
    #keyingtries=3
    fragmentation=yes
    also=ikecommon
conn L2TP-PSK-NAT
    leftfirewall=yes
    rightfirewall=yes
    also=L2TP-PSK-noNAT
conn L2TP-PSK-noNAT
    keyexchange=ikev1
    type=transport
    authby=psk
    keyingtries=3
    rekey=no
    leftprotoport=17/1701
    rightprotoport=17/%any
    ike="aes256-sha1-modp2048!"
    esp="aes-sha1!"

Edit strongSwan configuration file:

vi /etc/strongswan.conf

Add below

duplicheck.enable = no 
install_virtual_ip = yes 
i_dont_care_about_security_and_use_aggressive_mode_psk = yes 
dns1 = 8.8.8.8 
dns2 = 8.8.4.4

to /etc/strongswan.conf charon block:

charon {
# number of worker threads in charon threads = 16
**INSERT HERE**
# send strongswan vendor ID?
# send_vendor_id = yes 

Edit file for public key:

vi /etc/ipsec.secrets

Add below to /etc/ipsec.secrets:

: PSK "YOURPSKKEY" 
YOURUSERNAME : XAUTH "YOURPASSWORD"

Replace YOURPSKKEY YOURUSERNAME YOURPASSWORD with your entries.

Add following rules to iptables:

iptables -A INPUT -p esp -j ACCEPT iptables -A INPUT -p udp --dport 500 -j ACCEPT iptables -A INPUT -p udp --dport 4500 -j ACCEPT

If your iptables supports MASQUERADE, then:

iptables -t nat -A POSTROUTING -s 10.0.0.0/8 -o venet0 -j MASQUERADE

Otherwise:

iptables -A POSTROUTING -t nat -s 10.0.0.0/8 -j SNAT --to-source YOURSERVERIP

Replace YOURSERVERIP with your server ip address.

If you want enable internal communication between VPN clients, add:

iptables -A FORWARD -s 10.0.0.0/8 -j ACCEPT

Start strongSwan:

/usr/local/sbin/ipsec start

If you want strongSwan running in the foreground, then use:

/usr/local/sbin/ipsec start --nofork