サーバーが側で海外IPなどのアクセス制限がある場合、googleのクローラーIPが海外のため、はじかれることがある。

.htaccessに許可リストを登録することによって解決する。

allow from 2001:4860:4801:10::/64
allow from 2001:4860:4801:11::/64
allow from 2001:4860:4801:12::/64
allow from 2001:4860:4801:13::/64
allow from 2001:4860:4801:14::/64
allow from 2001:4860:4801:15::/64
allow from 2001:4860:4801:16::/64
allow from 2001:4860:4801:17::/64
allow from 2001:4860:4801:18::/64
allow from 2001:4860:4801:19::/64
allow from 2001:4860:4801:1a::/64
allow from 2001:4860:4801:1b::/64
allow from 2001:4860:4801:1c::/64
allow from 2001:4860:4801:1d::/64
allow from 2001:4860:4801:20::/64
allow from 2001:4860:4801:21::/64
allow from 2001:4860:4801:22::/64
allow from 2001:4860:4801:23::/64
allow from 2001:4860:4801:24::/64
allow from 2001:4860:4801:25::/64
allow from 2001:4860:4801:26::/64
allow from 2001:4860:4801:27::/64
allow from 2001:4860:4801:28::/64
allow from 2001:4860:4801:29::/64
allow from 2001:4860:4801:2::/64
allow from 2001:4860:4801:2a::/64
allow from 2001:4860:4801:2b::/64
allow from 2001:4860:4801:2c::/64
allow from 2001:4860:4801:2d::/64
allow from 2001:4860:4801:2e::/64
allow from 2001:4860:4801:2f::/64
allow from 2001:4860:4801:30::/64
allow from 2001:4860:4801:31::/64
allow from 2001:4860:4801:32::/64
allow from 2001:4860:4801:33::/64
allow from 2001:4860:4801:34::/64
allow from 2001:4860:4801:35::/64
allow from 2001:4860:4801:36::/64
allow from 2001:4860:4801:37::/64
allow from 2001:4860:4801:38::/64
allow from 2001:4860:4801:39::/64
allow from 2001:4860:4801:3::/64
allow from 2001:4860:4801:3a::/64
allow from 2001:4860:4801:3b::/64
allow from 2001:4860:4801:3c::/64
allow from 2001:4860:4801:3d::/64
allow from 2001:4860:4801:3e::/64
allow from 2001:4860:4801:40::/64
allow from 2001:4860:4801:41::/64
allow from 2001:4860:4801:42::/64
allow from 2001:4860:4801:43::/64
allow from 2001:4860:4801:44::/64
allow from 2001:4860:4801:45::/64
allow from 2001:4860:4801:46::/64
allow from 2001:4860:4801:47::/64
allow from 2001:4860:4801:48::/64
allow from 2001:4860:4801:49::/64
allow from 2001:4860:4801:4a::/64
allow from 2001:4860:4801:50::/64
allow from 2001:4860:4801:51::/64
allow from 2001:4860:4801:53::/64
allow from 2001:4860:4801:54::/64
allow from 2001:4860:4801:60::/64
allow from 2001:4860:4801:61::/64
allow from 2001:4860:4801:62::/64
allow from 2001:4860:4801:63::/64
allow from 2001:4860:4801:64::/64
allow from 2001:4860:4801:65::/64
allow from 2001:4860:4801:66::/64
allow from 2001:4860:4801:67::/64
allow from 2001:4860:4801:68::/64
allow from 2001:4860:4801:69::/64
allow from 2001:4860:4801:6a::/64
allow from 2001:4860:4801:6b::/64
allow from 2001:4860:4801:6c::/64
allow from 2001:4860:4801:6d::/64
allow from 2001:4860:4801:6e::/64
allow from 2001:4860:4801:6f::/64
allow from 2001:4860:4801:70::/64
allow from 2001:4860:4801:71::/64
allow from 2001:4860:4801:72::/64
allow from 2001:4860:4801:73::/64
allow from 2001:4860:4801:74::/64
allow from 2001:4860:4801:75::/64
allow from 2001:4860:4801:76::/64
allow from 2001:4860:4801:77::/64
allow from 2001:4860:4801:80::/64
allow from 2001:4860:4801:81::/64
allow from 2001:4860:4801:82::/64
allow from 2001:4860:4801:83::/64
allow from 2001:4860:4801:84::/64
allow from 2001:4860:4801:85::/64
allow from 2001:4860:4801:86::/64
allow from 2001:4860:4801:87::/64
allow from 2001:4860:4801:88::/64
allow from 2001:4860:4801:90::/64
allow from 2001:4860:4801:91::/64
allow from 2001:4860:4801:92::/64
allow from 2001:4860:4801:93::/64
allow from 2001:4860:4801::/64
allow from 2001:4860:4801:c::/64
allow from 2001:4860:4801:f::/64
allow from 192.178.5.0/27
allow from 34.100.182.96/28
allow from 34.101.50.144/28
allow from 34.118.254.0/28
allow from 34.118.66.0/28
allow from 34.126.178.96/28
allow from 34.146.150.144/28
allow from 34.147.110.144/28
allow from 34.151.74.144/28
allow from 34.152.50.64/28
allow from 34.154.114.144/28
allow from 34.155.98.32/28
allow from 34.165.18.176/28
allow from 34.175.160.64/28
allow from 34.176.130.16/28
allow from 34.22.85.0/27
allow from 34.64.82.64/28
allow from 34.65.242.112/28
allow from 34.80.50.80/28
allow from 34.88.194.0/28
allow from 34.89.10.80/28
allow from 34.89.198.80/28
allow from 34.96.162.48/28
allow from 35.247.243.240/28
allow from 66.249.64.0/27
allow from 66.249.64.128/27
allow from 66.249.64.160/27
allow from 66.249.64.192/27
allow from 66.249.64.224/27
allow from 66.249.64.32/27
allow from 66.249.64.64/27
allow from 66.249.64.96/27
allow from 66.249.65.0/27
allow from 66.249.65.128/27
allow from 66.249.65.160/27
allow from 66.249.65.192/27
allow from 66.249.65.224/27
allow from 66.249.65.32/27
allow from 66.249.65.64/27
allow from 66.249.65.96/27
allow from 66.249.66.0/27
allow from 66.249.66.128/27
allow from 66.249.66.192/27
allow from 66.249.66.32/27
allow from 66.249.66.64/27
allow from 66.249.66.96/27
allow from 66.249.68.0/27
allow from 66.249.68.32/27
allow from 66.249.68.64/27
allow from 66.249.69.0/27
allow from 66.249.69.128/27
allow from 66.249.69.160/27
allow from 66.249.69.192/27
allow from 66.249.69.224/27
allow from 66.249.69.32/27
allow from 66.249.69.64/27
allow from 66.249.69.96/27
allow from 66.249.70.0/27
allow from 66.249.70.128/27
allow from 66.249.70.160/27
allow from 66.249.70.192/27
allow from 66.249.70.224/27
allow from 66.249.70.32/27
allow from 66.249.70.64/27
allow from 66.249.70.96/27
allow from 66.249.71.0/27
allow from 66.249.71.128/27
allow from 66.249.71.160/27
allow from 66.249.71.192/27
allow from 66.249.71.224/27
allow from 66.249.71.32/27
allow from 66.249.71.64/27
allow from 66.249.71.96/27
allow from 66.249.72.0/27
allow from 66.249.72.128/27
allow from 66.249.72.160/27
allow from 66.249.72.192/27
allow from 66.249.72.224/27
allow from 66.249.72.32/27
allow from 66.249.72.64/27
allow from 66.249.72.96/27
allow from 66.249.73.0/27
allow from 66.249.73.128/27
allow from 66.249.73.160/27
allow from 66.249.73.192/27
allow from 66.249.73.224/27
allow from 66.249.73.32/27
allow from 66.249.73.64/27
allow from 66.249.73.96/27
allow from 66.249.74.0/27
allow from 66.249.74.128/27
allow from 66.249.74.32/27
allow from 66.249.74.64/27
allow from 66.249.74.96/27
allow from 66.249.75.0/27
allow from 66.249.75.128/27
allow from 66.249.75.160/27
allow from 66.249.75.192/27
allow from 66.249.75.224/27
allow from 66.249.75.32/27
allow from 66.249.75.64/27
allow from 66.249.75.96/27
allow from 66.249.76.0/27
allow from 66.249.76.128/27
allow from 66.249.76.160/27
allow from 66.249.76.192/27
allow from 66.249.76.224/27
allow from 66.249.76.32/27
allow from 66.249.76.64/27
allow from 66.249.76.96/27
allow from 66.249.77.0/27
allow from 66.249.77.128/27
allow from 66.249.77.160/27
allow from 66.249.77.192/27
allow from 66.249.77.32/27
allow from 66.249.77.64/27
allow from 66.249.77.96/27
allow from 66.249.79.0/27
allow from 66.249.79.128/27
allow from 66.249.79.160/27
allow from 66.249.79.192/27
allow from 66.249.79.224/27
allow from 66.249.79.32/27
allow from 66.249.79.64/27
allow from 66.249.79.96/27

クローラー系の海外サービスも同じでアクセス排除される。
以下は、AhrefsのIPリスト

ahrefs ip list

allow from 54.36.148.0/24
allow from 54.36.149.0/24
allow from 195.154.122.0/24
allow from 195.154.123.0/24
allow from 195.154.126.0/24
allow from 195.154.127.0/24
allow from 51.222.253.0/26
allow from 168.119.64.245
allow from 168.119.64.246
allow from 168.119.64.247
allow from 168.119.64.248
allow from 168.119.64.249
allow from 168.119.64.250
allow from 168.119.64.251
allow from 168.119.64.252
allow from 168.119.64.253
allow from 168.119.64.254
allow from 168.119.65.107
allow from 168.119.65.108
allow from 168.119.65.109
allow from 168.119.65.110
allow from 168.119.65.111
allow from 168.119.65.112
allow from 168.119.65.113
allow from 168.119.65.114
allow from 168.119.65.115
allow from 168.119.65.116
allow from 168.119.65.117
allow from 168.119.65.118
allow from 168.119.65.119
allow from 168.119.65.120
allow from 168.119.65.121
allow from 168.119.65.122
allow from 168.119.65.123
allow from 168.119.65.124
allow from 168.119.65.125
allow from 168.119.65.126
allow from 168.119.65.43
allow from 168.119.65.44
allow from 168.119.65.45
allow from 168.119.65.46
allow from 168.119.65.47
allow from 168.119.65.48
allow from 168.119.65.49
allow from 168.119.65.50
allow from 168.119.65.51
allow from 168.119.65.52
allow from 168.119.65.53
allow from 168.119.65.54
allow from 168.119.65.55
allow from 168.119.65.56
allow from 168.119.65.57
allow from 168.119.65.58
allow from 168.119.65.59
allow from 168.119.65.60
allow from 168.119.65.61
allow from 168.119.65.62
allow from 168.119.68.117
allow from 168.119.68.118
allow from 168.119.68.119
allow from 168.119.68.120
allow from 168.119.68.121
allow from 168.119.68.122
allow from 168.119.68.123
allow from 168.119.68.124
allow from 168.119.68.125
allow from 168.119.68.126
allow from 168.119.68.171
allow from 168.119.68.172
allow from 168.119.68.173
allow from 168.119.68.174
allow from 168.119.68.175
allow from 168.119.68.176
allow from 168.119.68.177
allow from 168.119.68.178
allow from 168.119.68.179
allow from 168.119.68.180
allow from 168.119.68.181
allow from 168.119.68.182
allow from 168.119.68.183
allow from 168.119.68.184
allow from 168.119.68.185
allow from 168.119.68.186
allow from 168.119.68.187
allow from 168.119.68.188
allow from 168.119.68.189
allow from 168.119.68.190
allow from 168.119.68.235
allow from 168.119.68.236
allow from 168.119.68.237
allow from 168.119.68.238
allow from 168.119.68.239
allow from 168.119.68.240
allow from 168.119.68.241
allow from 168.119.68.242
allow from 168.119.68.243
allow from 168.119.68.244
allow from 168.119.68.245
allow from 168.119.68.246
allow from 168.119.68.247
allow from 168.119.68.248
allow from 168.119.68.249
allow from 168.119.68.250
allow from 168.119.68.251
allow from 168.119.68.252
allow from 168.119.68.253
allow from 168.119.68.254