Code Snippets

常用的代码, 方面复制粘贴

Python

matplotlib

在服务器上由于没有 display 设备, 因此, savefig 的时候会出错, 解决方法如下:

1
2
import matplotlib
matplotlib.use('Agg')

post form data

1
2
3
4
url = "http://10.84.145.69:8080"
img = {"image": open("./example.jpeg", "rb")}
res = requests.post(url, files=img)
print res.text)

FCN 通过 tornado 提供服务的一个例子

没有找到全局初始化的方法, 只有每个响应初始化一次的方法

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
class FCN(tornado.web.RequestHandler):
def initialize(self, sess, pred, image):
self.sess = sess
self.pred = pred
self.image = image
def predict(self, mat):
preds = self.sess.run(self.pred, feed_dict={self.image: mat})
preds = postprocess(mat, preds)
return preds
def get_contour(self, mat):
nb_components, output, stats, centroids = cv2.connectedComponentsWithStats(mat, connectivity=8)
sizes = stats[1:, -1]
nb_components = nb_components - 1
min_size = 32
img2 = np.zeros((output.shape), dtype=np.uint8)
for i in range(0, nb_components):
if sizes[i] >= min_size:
img2[output == i + 1] = 255
mat2, contours, hierarchy = cv2.findContours(img2, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
return [np.squeeze(c).tolist() for c in contours]
def post(self):
fileinfo = self.request.files['image'][0]
fname = fileinfo['filename']
extn = os.path.splitext(fname)[-1]
cname = str(uuid.uuid4())+extn
with open(os.path.join(cur_path, __UPLOAD__, cname), 'w') as fh:
fh.write(fileinfo['body'])
filedata = fileinfo["body"]
mat = cv2.imdecode(np.asarray(bytearray(filedata), dtype=np.uint8), 1)
mat = cv2.resize(mat, (conf["width"], conf["height"]))
mat = np.array([mat])
res = self.predict(mat)
res = np.squeeze(res)
res = res.astype(np.uint8)
res = self.get_contour(res)
self.write(json.dumps({"result": res}))
if __name__ == "__main__":
os.environ["CUDA_VISIBLE_DEVICES"] = conf["CUDA"]
if sys.argv[1] == "train":
train()
elif sys.argv[1] == "test":
predict(conf["model_dir"])
else:
sess = tf.Session()
image = tf.placeholder(tf.float32, shape=[1, conf["height"], conf["width"], 3], name="input_image")
pred, _ = inference(image, 1)
saver = tf.train.Saver()
saver.restore(sess, conf["model_dir"])
logger.info("model ready")
application = tornado.web.Application([(r"/", FCN, dict(sess=sess, pred=pred, image=image))], debug=False)
application.listen(8080)
tornado.ioloop.IOLoop.instance().start()

设置 UTF-8 编码

1
2
3
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

调用系统命令

1
2
import subprocess
subprocess.call(["ls", "-l", "/home/"])

配置日志输出格式

1
2
3
4
5
6
7
8
9
10
11
import logging
logging.basicConfig(
level=logging.DEBUG,
# asctime: 日期+时间
# filename: 输出内容的代码所在的文件
# funcName: 输出内容的代码所在的函数
# lineno: 输出内容的代码所在的行数
# level: 输出内容的 level
# message: 输出的内容
format="%(asctime)s %(filename)s %(funcName)s(): %(lineno)i: %(levelname)s: %(message)s", )
logger = logging.getLogger(__name__)

遍历目录下面的所有指定类型的文件,包括子目录的文件

1
2
3
4
5
6
7
8
9
10
def mk_list(target=None):
"""create a list file of all images"""
matches = []
for root, dirnames, filenames in os.walk(target):
for filename in fnmatch.filter(filenames, '*.jpg'):
matches.append(os.path.join(root, filename))
f = open(__imglist__, w)
for img in matches:
f.write(img + '\n')
return matches

提供POST服务

需要接收上传的文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
import tornado
import tornado.web
import tornado.ioloop
import os
import uuid
class Upload(tornado.web.RequestHandler):
def post(self):
fileinfo = self.request.files['image'][0]
fname = fileinfo['filename']
extn = os.path.splitext(fname)[1]
cname = str(uuid.uuid4())+extn
fh = open(__UPLOADS__+cname, 'w')
fh.write(fileinfo['body'])
print os.path.abspath(__UPLOADS__+cname)
ret = web_demo([os.path.abspath(__UPLOADS__ + cname)])
self.write(ret)
# 返回/渲染 html 页面
class Userform(tornado.web.RequestHandler):
def get(self):
self.render("fileuploadform.html")
application = tornado.web.Application([
(r"/", Userform),
(r"/upload", Upload),
(r'/images/(.*)', tornado.web.StaticFileHandler,{'path': __static__path__}) # 直接访问获取图片
], debug= True)
if __name__ == "__main__":
application.listen(8888)
tornado.ioloop.IOLoop.instance().start()

不需要接收文件

1
2
3
4
5
6
7
8
9
10
11
12
13
import tornado.web
import tornado.ioloop
import json
import os, sys
class GetTaskHandler(tornado.web.RequestHandler):
def post(self):
parsed_json = json.loads(self.request.body)
task_type = parsed_json['task_type']
row = self.GetTask(task_req=task_type)
if row:
self.write(row)
else:
self.write('no more data')

多进程

不再推荐的方法, 不能捕获 Ctrl-C 信号

1
2
import multiprocessing
multiprocessing.Pool(processes=multiprocessing.cpu_count()).map(func, list) # func 是具体处理函数, list 是可迭代对象

推荐使用的方法

1
multiprocessing.Pool(processes=multiprocessing.cpu_count()).map_async(func, list).get(3600) # func 是具体处理函数, list 是可迭代对象

获取当前文件所在的路径

1
cur_path = os.path.abspath(os.path.dirname(__file__))

使用utf-8编码

1
2
#!/usr/bin/env python
# -*- coding: utf-8 -*-

命令行解析

1
2
3
parser = argparse.ArgumentParser(description='train an image classifer on cifar10')
parser.add_argument('--network', type=str, default='inception-bn-28-small', help = 'the cnn to use')
args = parser.parse_args()

MXNet

获取某些层的内部信息

1
2
3
4
symbol, arg_params, aux_params = mx.model.load_checkpoint("./model/Inception-7", 1)
symbol= symbol.get_internals()['flatten_output'] # 获取内部的某层
# allow_extra_params 必须要
new_model = mx.model.FeedForward(symbol,ctx=mx.gpu(), arg_params=arg_params, aux_params=aux_params, allow_extra_params=True)

渲染可视化网络结构

1
mx.visualization.plot_network(symbol).render(filename, cleanup=True)

OpenCV

Display 的图像可以随意改变大小

1
2
cv2.namedWindow(windowname, cv2.WINDOW_FREERATIO) # cv2.WINDOW_FREERATIO 也可以是其它的值
cv2.imshow(windowname, mat)

cv2.imread(img, 0)cv2.cvtColor(mat, cv2.BGR2GRAY)效果是一样的
cv2.mean()返回的总是一个 4 元的数组, 所以, 在计算灰度图的时候, 要取第一个元素作为图像的均值.

彩色图像的直方图均衡化增强

在灰度图中由于是单通道, 而已直接对该通道做直方图均衡, 但是, 对于 RGB 图像, 由于是 3 个通道, 如果对 3 个通道分别做直方图均衡然后组合回去的话会影响了颜色效果, 因此, 对于彩色图像常用的做法是把图像转换到 YUV 空间, 然后, 在该颜色空间中只对 intensity value 直方图均衡, 也就是 YUV 空间的 Y 通道.

1
2
3
4
5
6
7
8
9
10
11
import cv2
import numpy as np
img = cv2.imread('input.jpg')
img_yuv = cv2.cvtColor(img, cv2.COLOR_BGR2YUV)
# equalize the histogram of the Y channel
img_yuv[:,:,0] = cv2.equalizeHist(img_yuv[:,:,0])
# convert the YUV image back to RGB format
img_output = cv2.cvtColor(img_yuv, cv2.COLOR_YUV2BGR)
cv2.imshow('Color input image', img)
cv2.imshow('Histogram equalized', img_output)
cv2.waitKey(0)

nginx

配置 nginx 作为反向代理实现负载均衡

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
upstream server1 {
# change this configure to conrresponding ports
server 127.0.0.1:8001;
server 127.0.0.1:8002;
server 127.0.0.1:8003;
server 127.0.0.1:8004;
}
server {
listen 8000;
root /path/to/server1; # change this config to the corresponding path
location / {
proxy_pass http://server1;
proxy_connect_timeout 60s;
}
}
#---------------------------------------------------------
upstream server2 {
# change this configure to conrresponding ports
server 127.0.0.1:9001;
server 127.0.0.1:9002;
server 127.0.0.1:9003;
server 127.0.0.1:9004;
}
server {
listen 9000;
root /path/to/server2; # change this config to the corresponding path
location / {
proxy_pass http://server2;
proxy_connect_timeout 60s;
}
}

HTML

ajax 使用例子

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>图像搜索</title>
<!-- Bootstrap core CSS -->
<link href="//cdn.bootcss.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet">
<!-- Bootstrap theme -->
<!-- <link href="../../dist/css/bootstrap-theme.min.css" rel="stylesheet"> -->
<!-- Custom styles for this template -->
<!-- <link href="jumbotron.css" rel="stylesheet"> -->
</head>
<body>
<div class="container theme-showcase" role="main">
<div class="jumbotron">
<center>
<h1>图像搜索</h1>
<p>图像搜索Demo</p>
</center>
</div>
<!-- </div> -->
<script src="https://ajax.googleapis.bootcss.com/ajax/libs/jquery/1.12.0/jquery.min.js">
</script>
<form action="/" method="post" enctype="multipart/form-data">
<input type="file" name="file" id="file">
</form>
<br>
<button type="button" class="btn btn-primary" id="upload">上传图片</button>
<br>
<div id="origin"></div>
<div id="escape"></div>
<div class="page-header">
<h1>搜索结果</h1>
</div>
<div id="result">
</div>
<script>
$(document).ready(function(){
$("#upload").click(function(){
var formData=new FormData();
formData.append('file', $('#file')[0].files[0]);
$.ajax({
url: "/",
type: "POST",
data: formData,
processData: false,
contentType: false,
success: function(data){
var imgs=JSON.parse(data)
document.getElementById("origin").innerHTML ="<center><img src=uploads/"+imgs.upload+" class=img-thumbnail width=256></center>"
document.getElementById("escape").innerHTML = imgs.escape
var all_img=""
for (img in imgs.result){
if (img % 2 < 0){
}
all_img+="<img src=images/"+imgs.result[img]+" class=img-thumbnail width=256>"
}
document.getElementById("result").innerHTML =all_img
}
});
});
});
</script>
</div>
</body>
</html>

html5 插入视频

1
2
3
4
<video width="320" height="240" controls>
<source src="/content/videos/3.mp4" type="video/mp4">
Your browser does not support the video tag.
</video>