1.背景

首先，这次的爬虫的对象是针对美拍的视频。
那么，如何去爬美拍的视频呢？
然后我们的目标是：
1：单独一个视频去爬
2：已经知道美拍用户的id，根据用户id去获取改用户的全部视频

2.单独获取一个视频

a.如何去获取一个视频呢？这个时候可以查看页面的源码。 例如 http://www.meipai.com/media/484461748 这个页面的视频。如何去获取这个页面的视频的真实video地址呢？
答案是：查看源码

<meta charset="UTF-8">
        <title>浅浅的唱歌给自己听#60秒美拍##音乐# - 田紫紫Tina的美拍</title>    
    <meta name="description" content="浅浅的唱歌给自己听#60秒美拍##音乐#由美拍用户田紫紫Tina发布分享，美拍女神频道推荐。">

    <meta name="keywords" content="60秒美拍,音乐,田紫紫Tina"/>
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
            <!-- 增加og相关标签属性，便于facebook抓取页面视频信息 start-->
        <meta content="video.other" property="og:type">
        <meta content="http://www.meipai.com/media/484461748" property="og:url">
        <meta content="浅浅的唱歌给自己听#60秒美拍##音乐#" property="og:title">
        <meta content="http://mvimg1.meitudata.com/56d6c580712008897.jpg" property="og:image">
        <meta content="浅浅的唱歌给自己听#60秒美拍##音乐#" property="og:description">
        <meta property="og:video:release_date" content="03-02" />
        <meta property="og:video:director" content="田紫紫Tina" />
         
            <meta property="og:video:tag" content="60秒美拍,音乐" />
                
        <meta content="video/mp4" property="og:video:type">
        <meta content="https://mvvideo5.meitudata.com/56d6fa507ac516008.mp4" property="og:video:url">
        <meta content="https://mvvideo5.meitudata.com/56d6fa507ac516008.mp4" property="og:video:secure_url">
        <meta content="480" property="og:video:width">
        <meta content="480" property="og:video:height">

在一开始这个页面，你会发现真实的视频地址就是 标签里面。那么现在要实现的就是写个网络请求，把页面捉取，然后写个正则表达式，想要的数据获取。
当知道完思路后，那么就开始写代码了。

#coding:utf-8
#用于获取网页内的视频地址
#这个是获取 美拍视频地址
import urllib2
import json
import urllib
import urlparse
from bs4 import BeautifulSoup
import json
import re
base_url ="http://www.meipai.com/media/484461748"
  
# import sys  
# reload(sys)  
# sys.setdefaultencoding('utf8')   
request = urllib2.Request(base_url)
response = urllib2.urlopen(request, data=None)
# print response.read()
reader = response.read()
# print response.read()
# print reader
soup = BeautifulSoup(reader,"lxml")
# print soup.text
# print images[0]['content']
vidoes = soup.find_all("meta",{'property':'og:video:url'})
print vidoes
print str(vidoes[0]['content'])

#正则表达式
rege = "<meta content=\"https://.*mp4"
items = re.findall(rege, reader)
print items

下面的是运行结果：  
[<meta content="https://mvvideo5.meitudata.com/56d6fa507ac516008.mp4" property="og:video:url"/>]
https://mvvideo5.meitudata.com/56d6fa507ac516008.mp4
['<meta content="https://mvvideo5.meitudata.com/56d6fa507ac516008.mp4', '<meta content="https://mvvideo5.meitudata.com/56d6fa507ac516008.mp4'] 


def show
  @widget = Widget(params[:id])
  respond_to do |format|
    format.html # show.html.erb
    format.json { render json: @widget }
  end
end

那么简单的获取一个视频的爬虫就得到了，那么根据user ID获取视频列表的方法呢？
下面详细讲解。

获取一个用户的全部视频

1：首先明确的是，美拍的用户跟url的关系
就好像 http://www.meipai.com/user/10810495 这个htto链接一样。 user后面的部分可以看成是用户的id 那么这个时候可以利用http捉包工具，获取对应的api。 因为美拍的页面在向下滑动的时候，自动加载新的信息。可以确认使用了ajax的对应框架。
http://www.meipai.com/users/user_timeline?page=2&count=12&single_column=1&uid=35079656
就好像上面的url地址一样，最后返回的是json数据
例如：

最后想办法解析数据，然后就可以得到真实视频url的地址。那么最后就可以下载视频了。

#coding:utf-8
import urllib2
import json
import requests
import os
# page = 2
# num = 13
print "start"
name = "1047127268"#40882577
base_url = "http://www.meipai.com/users/user_timeline?page={page}&count=12&single_column=1&uid={name}".replace("{name}", name)
# url = base_url.replace("{page}", str(page))
# req = urllib2.urlopen(url)
# body =  req.read()
# js = json.loads(body)
# medias =  js.get("medias")

os.system("mkdir mp3/"+name)
print "test"
mp4_url = []
def getMp4Url(page):
    url = base_url.replace("{page}",str(page))
    req = urllib2.urlopen(url)
    body =  req.read()
    js = json.loads(body)
    # print body
    medias =  js.get("medias")
    for item in medias:
        video = item.get("video")
        mp4_url.append(video)
    if len(medias) == 12:
        getMp4Url(page+1)

def downvideo(url,i):
    s = requests.session()
    # s.config['keep_alive'] = False
    s.headers['keep_alive'] = True
    s.headers['Connection']='Keep-Alive'
    s.headers['User-Agent'] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36'}"
    s.headers['X-Requested-With']="ShockwaveFlash/20.0.0.267"
    s.headers['Cookie']="xa=xa"
    s.headers['Accept-Language']='zh-CN,zh;q=0.8,en;q=0.6'
    s.headers['Accept-Encoding'] ='gzip, deflate, sdch'
    # s.headers['Referer'] = 'http://www.flv.tv/q{1}'.replace("{1}",str(i))
    # s.headers['Host']='n.syasn.com'
    r = s.get(url)
    # print r.headers
    with open ("./mp3/"+name+"/"+str(i)+".mp4","wb") as code:
        code.write(r.content)
    print "index :",i
getMp4Url(1)
print "len",len(mp4_url)
# os.system("command")

fi = open("./mp3/"+name+".txt", mode="w")
for item in mp4_url:
    fi.writelines(item+"\n")
fi.close()
num = 1
for item in mp4_url:
    print num,item 
    # downvideo(item, num)
    num = num+1
print "end"
# print len(medias)
# for i in medias:
    # video_url =  i.get("video") 
    # downvideo(video_url, num)
    # num = num+1
    # print video_url

运行结果：

1 http://mvvideo1.meitudata.com/56e05e81a156f8543.mp4
2 http://mvvideo2.meitudata.com/56ba89507b7a24113.mp4
3 http://mvvideo2.meitudata.com/56b68fcfe0c1f3247.mp4
4 http://mvvideo1.meitudata.com/56b69dbe763009024.mp4
5 http://mvvideo2.meitudata.com/56b53681ba7fe4708.mp4
6 http://mvvideo2.meitudata.com/56b3dc698ab77870.mp4
7 http://mvvideo1.meitudata.com/56b2b460f05ac8358.mp4
8 http://mvvideo2.meitudata.com/56b207ba749973068.mp4
9 http://mvvideo1.meitudata.com/56b1d377a8d9a2375.mp4
10 http://mvvideo2.meitudata.com/56b16b5f8f0722203.mp4
11 http://mvvideo2.meitudata.com/56b02eb7d97ae9729.mp4
12 http://mvvideo1.meitudata.com/5694827a60e512951.mp4
13 http://mvvideo2.meitudata.com/569037c802f0b4335.mp4
14 http://mvvideo2.meitudata.com/568e5c16da05b2425.mp4
15 http://mvvideo1.meitudata.com/568d07362646c8484.mp4
16 http://mvvideo2.meitudata.com/566b035069c593107.mp4
17 http://mvvideo2.meitudata.com/5669c74360e8f9059.mp4
18 http://mvvideo1.meitudata.com/56692d76694e81818.mp4
19 http://mvvideo2.meitudata.com/5668b3993dba68223.mp4
20 http://mvvideo2.meitudata.com/566af513f3a578748.mp4
21 http://mvvideo1.meitudata.com/566b02ff9b0354731.mp4
22 http://mvvideo2.meitudata.com/56661fb8787342002.mp4
23 http://mvvideo2.meitudata.com/56680a90c34961786.mp4
24 http://mvvideo2.meitudata.com/566827badea67914.mp4
25 http://mvvideo1.meitudata.com/56684d6d7acc46645.mp4
26 http://mvvideo1.meitudata.com/5661b7a8e29cd9413.mp4
27 http://mvvideo1.meitudata.com/5661b23f4afad897.mp4
28 http://mvvideo1.meitudata.com/56694af0719933371.mp4
29 http://mvvideo1.meitudata.com/565836e8b699a108.mp4
30 http://mvvideo2.meitudata.com/5669d6a226ec16302.mp4
31 http://mvvideo2.meitudata.com/5669d880c5b518638.mp4
32 http://mvvideo2.meitudata.com/5669d8f1925e16318.mp4
33 http://mvvideo1.meitudata.com/5669d9ce21687448.mp4
34 http://mvvideo1.meitudata.com/5669e60b306251274.mp4
35 http://mvvideo2.meitudata.com/5669e72dccdcc8200.mp4
36 http://mvvideo2.meitudata.com/5669e707b5f0a3291.mp4
37 http://mvvideo2.meitudata.com/566a069f42b745119.mp4
38 http://mvvideo2.meitudata.com/565040b7953a71639.mp4
39 http://mvvideo1.meitudata.com/56503a8c28fbf5614.mp4
40 http://mvvideo2.meitudata.com/56502a51d96f11887.mp4
41 http://mvvideo1.meitudata.com/5650259251d1c6694.mp4
42 http://mvvideo1.meitudata.com/566a4b8aeba7e5522.mp4
43 http://mvvideo2.meitudata.com/564d861123f8f7577.mp4
44 http://mvvideo1.meitudata.com/564d7a4a4a22b3289.mp4
45 http://mvvideo1.meitudata.com/566a7bc57a3d93075.mp4
46 http://mvvideo1.meitudata.com/564cabd33cae49337.mp4
47 http://mvvideo2.meitudata.com/564b2ff06b598625.mp4
48 http://mvvideo1.meitudata.com/564afc48e9ff57990.mp4
49 http://mvvideo2.meitudata.com/564af6d5632ed5644.mp4
50 http://mvvideo2.meitudata.com/564cdd6e035196867.mp4
51 http://mvvideo2.meitudata.com/564cde1de660a3068.mp4
52 http://mvvideo2.meitudata.com/564ce979d8b772775.mp4
53 http://mvvideo1.meitudata.com/5643c0f9db4712070.mp4
54 http://mvvideo1.meitudata.com/564d2735e188a4376.mp4
55 http://mvvideo2.meitudata.com/564d270a818a04718.mp4
56 http://mvvideo1.meitudata.com/56422da4776409177.mp4
57 http://mvvideo2.meitudata.com/5637acda9945b5991.mp4
58 http://mvvideo1.meitudata.com/563c8d0b8e0031466.mp4
59 http://mvvideo2.meitudata.com/563c8d117e2d4296.mp4
60 http://mvvideo2.meitudata.com/564dc502f02f1675.mp4
61 http://mvvideo2.meitudata.com/5633d7b2d51227818.mp4
62 http://mvvideo1.meitudata.com/5633d7c0f2bef2814.mp4
63 http://mvvideo1.meitudata.com/5633faf91dc202085.mp4
64 http://mvvideo2.meitudata.com/5634324d6efad6440.mp4
65 http://mvvideo1.meitudata.com/56343e598afac7683.mp4
66 http://mvvideo1.meitudata.com/56343e4933f108664.mp4
67 http://mvvideo2.meitudata.com/56343e438d3b69953.mp4
68 http://mvvideo2.meitudata.com/56343e4abe10a3502.mp4
69 http://mvvideo1.meitudata.com/563441f0bae67919.mp4
70 http://mvvideo2.meitudata.com/56344971b1a336725.mp4
71 http://mvvideo2.meitudata.com/5634498f99cf09863.mp4
72 http://mvvideo1.meitudata.com/563449a08696a3620.mp4
73 http://mvvideo1.meitudata.com/56344d3e291467064.mp4
74 http://mvvideo1.meitudata.com/56344d26ae1018661.mp4
75 http://mvvideo1.meitudata.com/56344d6b038db9093.mp4
76 http://mvvideo10.meitudata.com/560124e8ea904691.mp4

注意，注意，有时候规则会改变。需要适当的维护