1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
|
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import django_filters
import shortuuid
from django.contrib.auth.hashers import make_password
from rest_framework.pagination import PageNumberPagination
from .exception import TestDataUploadError
from .filters import TestRecordListFilter
from .models import UserMachine, TestCategory, TestBranch
from pgperffarm.settings import DB_ENUM
from user_operation.views import UserMachinePermission
from .serializer import MachineHistoryRecordSerializer, TestStatusRecordListSerializer, TestBranchSerializer, \
CreateTestResultSerializer, CreatePGInfoSerializer
from .serializer import TestRecordListSerializer, TestRecordDetailSerializer, LinuxInfoSerializer, MetaInfoSerializer, \
PGInfoSerializer, CreateTestRecordSerializer, CreateTestDateSetSerializer, TestResultSerializer
from rest_framework.decorators import api_view, permission_classes
from rest_framework.response import Response
from rest_framework import mixins
from rest_framework import status
from rest_framework import viewsets
from .models import TestRecord
import json
class StandardResultsSetPagination(PageNumberPagination):
page_size = 20
page_size_query_param = 'page_size'
max_page_size = 100
class BigResultsSetPagination(PageNumberPagination):
page_size = 1000
page_size_query_param = 'page_size'
class TestBranchListViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
"""
List test records
"""
queryset = TestBranch.objects.all().order_by('branch_order','add_time')
serializer_class = TestBranchSerializer
pagination_class = BigResultsSetPagination
class TestRecordListViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
"""
List test records
"""
queryset = TestRecord.objects.all().order_by('add_time')
serializer_class = TestRecordListSerializer
pagination_class = StandardResultsSetPagination
filter_backends = (django_filters.rest_framework.DjangoFilterBackend,)
filter_class = TestRecordListFilter
class TestRecordListByBranchViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
"""
List test records (/status)
"""
queryset = TestRecord.objects.order_by('test_machine__alias__name','-add_time').distinct('test_machine__alias__name').all()
serializer_class = TestRecordListSerializer
pagination_class = StandardResultsSetPagination
filter_backends = (django_filters.rest_framework.DjangoFilterBackend,)
filter_class = TestRecordListFilter
# @api_view(['GET'])
# def GetStatusRecordList(request, format=None):
# """
# List lastest test records involve all branches
# """
#
# queryset = TestBranch.objects.all().order_by('branch_order').values_list('id','branch_name').annotate(num_records=Count('testrecord')).filter(num_records__gt=0)
# # print queryset # <QuerySet [(1, u'HEAD', 3), (2, u'10_STABLE', 2)]>
#
# ret = {'branch_num':queryset.__len__(),'result':[]}
# for branch_item in queryset:
#
# target_record = TestRecord.objects.filter(branch_id=branch_item[0]).order_by('test_machine_id','-add_time').distinct('test_machine_id').all()
# # print target_record # <QuerySet [(1, u'HEAD', 3), (2, u'10_STABLE', 2)]>
# data = TestRecordListSerializer(target_record,many=True)
# obj = {'branch':branch_item[1],'data':data.data}
# ret["result"].append(obj)
# # msg = 'ok!'
# return Response(ret, status=status.HTTP_201_CREATED)
class TestRecordDetailViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
"""
detail test records
"""
lookup_field = 'uuid'
queryset = TestRecord.objects.all().order_by('add_time')
serializer_class = TestRecordDetailSerializer
# pagination_class = StandardResultsSetPagination
class MachineHistoryRecordViewSet( mixins.RetrieveModelMixin, viewsets.GenericViewSet):
"""
machine info page
"""
lookup_field = 'machine_sn'
queryset = UserMachine.objects.all().order_by('add_time')
serializer_class = MachineHistoryRecordSerializer
# pagination_class = StandardResultsSetPagination
@api_view(['POST'])
@permission_classes((UserMachinePermission, ))
def TestRecordCreate(request, format=None):
"""
Receive data from client
"""
print(request.__str__())
data = request.data
print(type(data[0]))
json_data = json.dumps(data[0], encoding="UTF-8", ensure_ascii=False)
json_data = json.loads(json_data, encoding="UTF-8")
# obj = data[0].pgbench
# jsLoads = json.loads(data[0])
from django.db import transaction
try:
# todo get machine by token
secret = request.META.get("HTTP_AUTHORIZATION")
ret = UserMachine.objects.filter(machine_secret=secret, state=1).get()
test_machine = ret.id
if test_machine <= 0:
raise TestDataUploadError("The machine is unavailable.")
record_hash = make_password(str(json_data), 'pg_perf_farm')
# print(record_hash.__len__()) 77
r = TestRecord.objects.filter(hash=record_hash).count()
if r != 0:
raise TestDataUploadError("The same record already exists, please do not submit it twice.")
with transaction.atomic():
linux_data = json_data['linux']
linuxInfo = LinuxInfoSerializer(data=linux_data)
linuxInfoRet = None
if linuxInfo.is_valid():
linuxInfoRet = linuxInfo.save()
else:
msg = 'linuxInfo invalid'
raise TestDataUploadError(msg)
meta_data = json_data['meta']
metaInfo = MetaInfoSerializer(data=meta_data)
metaInfoRet = None
if metaInfo.is_valid():
metaInfoRet = metaInfo.save()
else:
msg = 'metaInfo invalid'
raise TestDataUploadError(msg)
pg_data = json_data['postgres']
branch_str = pg_data['branch']
if(branch_str == 'master'):
branch_str = 'HEAD'
branch = TestBranch.objects.filter(branch_name__iexact=branch_str,is_accept=True).get()
if not branch:
raise TestDataUploadError('The branch name is unavailable.')
commit = pg_data['commit']
pg_settings = pg_data['settings']
filtered=['checkpoint_timeout','work_mem','shared_buffers','maintenance_work_mem','max_wal_size','min_wal_size']
for item in filtered:
pg_settings[item] = pg_settings[item].encode('utf-8')
pg_settings[item] = filter(str.isdigit, pg_settings[item])
pg_settings['log_checkpoints'] = DB_ENUM['general_switch'][pg_settings['log_checkpoints']]
pgInfo = CreatePGInfoSerializer(data=pg_settings)
pgInfoRet = None
if pgInfo.is_valid():
pgInfoRet = pgInfo.save()
else:
msg = pgInfo.errors
raise TestDataUploadError(msg)
test_record_data = {
'pg_info': pgInfoRet.id,
'linux_info': linuxInfoRet.id,
'meta_info': metaInfoRet.id,
'test_machine': test_machine,
'test_desc': 'here is desc',
'meta_time': metaInfoRet.date,
'hash': record_hash,
'commit': commit,
'branch':branch.id,
'uuid': shortuuid.uuid()
}
testRecord = CreateTestRecordSerializer(data=test_record_data)
testRecordRet = None
if testRecord.is_valid():
testRecordRet = testRecord.save()
else:
msg = 'testRecord invalid'
print(testRecord.errors)
raise TestDataUploadError(msg)
pgbench = json_data['pgbench']
# print(type(ro))
ro = pgbench['ro']
for tag, tag_list in pgbench.iteritems():
test_cate = TestCategory.objects.get(cate_sn=tag)
if not test_cate:
continue
else:
print(test_cate.cate_name)
for scale, dataset_list in tag_list.iteritems():
print("ro[%s]=" % scale, dataset_list)
for client_num, dataset in dataset_list.iteritems():
print('std is:' + str(dataset['std']))
test_dataset_data = {
'test_record': testRecordRet.id,
'clients': client_num,
'scale': scale,
'std': dataset['std'],
'metric': dataset['metric'],
'median': dataset['median'],
'test_cate': test_cate.id,
# status,percentage will calc by receiver
'status': -1,
'percentage': 0.0,
}
testDateSet = CreateTestDateSetSerializer(data=test_dataset_data)
testDateSetRet = None
if testDateSet.is_valid():
print('dataset valid')
testDateSetRet = testDateSet.save()
else:
# print(testDateSet.errors)
msg = 'testDateSet invalid'
raise TestDataUploadError(msg)
test_result_list = dataset['results']
for test_result in test_result_list:
test_result_data = test_result
test_result_data['test_dataset'] = testDateSetRet.id
test_result_data['mode'] = DB_ENUM['mode'][test_result_data['mode']]
testResult = CreateTestResultSerializer(data=test_result_data)
testResultRet = None
if testResult.is_valid():
print('testResult valid')
testResultRet = testResult.save()
else:
# print(testResult.error_messages)
msg = testResult.error_messages
raise TestDataUploadError(msg)
except Exception as e:
msg = 'upload error:' + e.__str__()
# todo add log
return Response(msg, status=status.HTTP_202_ACCEPTED)
msg = 'upload success!'
return Response(msg, status=status.HTTP_201_CREATED)
|
