|
| 1 | +# -*- coding: utf-8 -*- |
| 2 | +''' |
| 3 | +© 2012-2013 eBay Software Foundation |
| 4 | +Authored by: Tim Keefer |
| 5 | +Licensed under CDDL 1.0 |
| 6 | +''' |
| 7 | + |
| 8 | +import os |
| 9 | +import sys |
| 10 | +from optparse import OptionParser |
| 11 | +import csv |
| 12 | +import json |
| 13 | +import requests |
| 14 | + |
| 15 | +sys.path.insert(0, '%s/../' % os.path.dirname(__file__)) |
| 16 | + |
| 17 | +from common import dump |
| 18 | + |
| 19 | +import ebaysdk |
| 20 | +from ebaysdk.finding import Connection as finding |
| 21 | +from ebaysdk.exception import ConnectionError, RequestPaginationError, PaginationLimit |
| 22 | + |
| 23 | + |
| 24 | +def init_options(): |
| 25 | + usage = "usage: %prog [options]" |
| 26 | + parser = OptionParser(usage=usage) |
| 27 | + |
| 28 | + parser.add_option("-d", "--debug", |
| 29 | + action="store_true", dest="debug", default=False, |
| 30 | + help="Enabled debugging [default: %default]") |
| 31 | + parser.add_option("-y", "--yaml", |
| 32 | + dest="yaml", default='ebay.yaml', |
| 33 | + help="Specifies the name of the YAML defaults file. [default: %default]") |
| 34 | + parser.add_option("-a", "--appid", |
| 35 | + dest="appid", default=None, |
| 36 | + help="Specifies the eBay application id to use.") |
| 37 | + parser.add_option("-s", "--store_name", |
| 38 | + dest="store_name", default=None, |
| 39 | + help="Store name") |
| 40 | + parser.add_option("-f", "--file", |
| 41 | + dest="input_file", default=None, |
| 42 | + help="Input file containing store names.") |
| 43 | + parser.add_option("-o", "--offset", |
| 44 | + dest="offset", default=0, |
| 45 | + help="Input file line offset.") |
| 46 | + parser.add_option("-e", "--line_end", |
| 47 | + dest="line_end", default=None, |
| 48 | + help="Input file lines.") |
| 49 | + |
| 50 | + (opts, args) = parser.parse_args() |
| 51 | + return opts, args |
| 52 | + |
| 53 | + |
| 54 | +def run(opts): |
| 55 | + |
| 56 | + data = None |
| 57 | + if opts.store_name: |
| 58 | + data = get_store_meta(opts.store_name) |
| 59 | + print(data) |
| 60 | + |
| 61 | + if opts.input_file: |
| 62 | + lines = [] |
| 63 | + with open(opts.input_file, newline='') as csvfile: |
| 64 | + for row1 in csv.reader(csvfile, delimiter=',', quotechar='"'): |
| 65 | + name = row1[1] |
| 66 | + desc = row1[2].replace('\n', '') |
| 67 | + logo = row1[3] |
| 68 | + sub = row1[4] |
| 69 | + lines.append([name, desc, logo, sub]) |
| 70 | + |
| 71 | + for row in lines[int(opts.offset):(int(opts.offset)+1000)]: |
| 72 | + print("(%s)" % row) |
| 73 | + |
| 74 | + if row[3] == 'http://': |
| 75 | + row[3] = None |
| 76 | + |
| 77 | + if row[0] and row[2] and row[3]: |
| 78 | + if record_exists(row[0]): |
| 79 | + print("skipping %s" % row[0]) |
| 80 | + continue |
| 81 | + |
| 82 | + try: |
| 83 | + data = get_store_meta(row[0]) |
| 84 | + except Exception as e: |
| 85 | + print("Exception %s" % e) |
| 86 | + |
| 87 | + if not data: |
| 88 | + continue |
| 89 | + |
| 90 | + data['store_logo'] = row[2] |
| 91 | + data['store_name'] = row[0] |
| 92 | + data['store_description'] = row[1] |
| 93 | + data['subscription_level'] = row[3] |
| 94 | + |
| 95 | + req = requests.post( |
| 96 | + 'http://elastic-1-2467465.lvs02.dev.ebayc3.com:9200/stores/storeMeta', |
| 97 | + data=json.dumps(data, sort_keys=True) |
| 98 | + ) |
| 99 | + print(req.text) |
| 100 | + |
| 101 | + else: |
| 102 | + pass |
| 103 | + |
| 104 | + |
| 105 | +def record_exists(store_name): |
| 106 | + query = { |
| 107 | + "query": { |
| 108 | + "match": { |
| 109 | + "store_name": { |
| 110 | + "query": store_name, |
| 111 | + "type": "phrase" |
| 112 | + } |
| 113 | + } |
| 114 | + } |
| 115 | + } |
| 116 | + |
| 117 | + query_req = requests.get('http://elastic-1-2467465.lvs02.dev.ebayc3.com:9200/stores/storeMeta/_search', |
| 118 | + data=json.dumps(query, sort_keys=True)) |
| 119 | + |
| 120 | + try: |
| 121 | + record_id = query_req.json()['hits']['hits'][0]['_id'] |
| 122 | + except Exception as e: |
| 123 | + record_id = None |
| 124 | + |
| 125 | + return record_id |
| 126 | + |
| 127 | + |
| 128 | +def get_store_meta(store_name): |
| 129 | + |
| 130 | + try: |
| 131 | + api = finding(debug=opts.debug, appid=opts.appid, |
| 132 | + config_file=opts.yaml) |
| 133 | + |
| 134 | + response = api.execute('findItemsIneBayStores', { |
| 135 | + 'storeName': store_name, |
| 136 | + 'outputSelector': [ |
| 137 | + 'CategoryHistogram', |
| 138 | + 'AspectHistogram', |
| 139 | + 'SellerInfo', |
| 140 | + 'StoreInfo', |
| 141 | + ]} |
| 142 | + ) |
| 143 | + |
| 144 | + if response.reply.ack != 'Success': |
| 145 | + return {} |
| 146 | + |
| 147 | + if int(response.reply.paginationOutput.totalEntries) <= 0: |
| 148 | + return {} |
| 149 | + |
| 150 | + data = { |
| 151 | + 'followers': 0, |
| 152 | + 'item_count': response.reply.paginationOutput.totalEntries, |
| 153 | + 'seller_name': response.reply.searchResult.item[0].sellerInfo.sellerUserName, |
| 154 | + 'store_url': response.reply.searchResult.item[0].storeInfo.storeURL, |
| 155 | + 'feedback_score': response.reply.searchResult.item[0].sellerInfo.feedbackScore, |
| 156 | + 'positive_feedback_percent': response.reply.searchResult.item[0].sellerInfo.positiveFeedbackPercent, |
| 157 | + 'top_rated_seller': response.reply.searchResult.item[0].sellerInfo.topRatedSeller, |
| 158 | + 'country_code': response.reply.searchResult.item[0].country, |
| 159 | + } |
| 160 | + |
| 161 | + agg_data = { |
| 162 | + 'cat_asp': {}, |
| 163 | + 'watch_count': 0, |
| 164 | + 'L0': [], |
| 165 | + 'L1': [], |
| 166 | + } |
| 167 | + |
| 168 | + dominate_l0_cat_count = 0 |
| 169 | + dominate_l1_cat_count = 0 |
| 170 | + |
| 171 | + for lev0 in response.reply.categoryHistogramContainer.categoryHistogram: |
| 172 | + agg_data['L0'].append({ |
| 173 | + 'category_id': lev0.categoryId, |
| 174 | + 'category_name': lev0.categoryName, |
| 175 | + 'item_count': lev0.count |
| 176 | + }) |
| 177 | + |
| 178 | + if int(lev0.count) > dominate_l0_cat_count: |
| 179 | + dominate_l0_cat_count = int(lev0.count) |
| 180 | + agg_data['dominate_l0_category_id'] = lev0.categoryId |
| 181 | + agg_data['dominate_l0_category_name'] = lev0.categoryName |
| 182 | + |
| 183 | + for lev1 in lev0.childCategoryHistogram: |
| 184 | + agg_data['L1'].append({ |
| 185 | + 'category_id': lev1.categoryId, |
| 186 | + 'category_name': lev1.categoryName, |
| 187 | + 'item_count': lev1.count |
| 188 | + }) |
| 189 | + |
| 190 | + if int(lev1.count) > dominate_l1_cat_count: |
| 191 | + dominate_l1_cat_count = int(lev1.count) |
| 192 | + agg_data['dominate_l1_category_id'] = lev1.categoryId |
| 193 | + agg_data['dominate_l1_category_name'] = lev1.categoryName |
| 194 | + |
| 195 | + |
| 196 | + |
| 197 | + for category_node in agg_data['L1']: |
| 198 | + category_id = category_node['category_id'] |
| 199 | + |
| 200 | + category_call = api.execute('findItemsIneBayStores', { |
| 201 | + 'storeName': store_name, |
| 202 | + 'categoryId': category_id, |
| 203 | + 'outputSelector': [ |
| 204 | + 'CategoryHistogram', |
| 205 | + 'AspectHistogram', |
| 206 | + 'SellerInfo', |
| 207 | + 'StoreInfo', |
| 208 | + ]} |
| 209 | + ) |
| 210 | + |
| 211 | + if category_call.reply.ack != 'Success': |
| 212 | + return {} |
| 213 | + |
| 214 | + if int(category_call.reply.paginationOutput.totalEntries) <= 0: |
| 215 | + return {} |
| 216 | + |
| 217 | + analyze_items(category_call.reply.searchResult.item, category_id, agg_data) |
| 218 | + |
| 219 | + try: |
| 220 | + while True: |
| 221 | + category_call2 = api.next_page() |
| 222 | + analyze_items(category_call2.reply.searchResult.item, category_id, agg_data) |
| 223 | + |
| 224 | + except PaginationLimit as e: |
| 225 | + pass |
| 226 | + |
| 227 | + dom_l1_asp = average_asp( |
| 228 | + agg_data['cat_asp'][agg_data['dominate_l1_category_id']] |
| 229 | + ) |
| 230 | + |
| 231 | + for category_node in agg_data['L1']: |
| 232 | + asp = average_asp(agg_data['cat_asp'][category_node['category_id']]) |
| 233 | + category_node.update({'asp': asp}) |
| 234 | + |
| 235 | + data.update({ |
| 236 | + 'L0': agg_data['L0'], |
| 237 | + 'L1': agg_data['L1'], |
| 238 | + 'watch_count': agg_data['watch_count'], |
| 239 | + 'postal_code': agg_data.get('postal_code', None), |
| 240 | + 'dominate_category_id': agg_data['dominate_l0_category_id'], |
| 241 | + 'dominate_category_name': agg_data['dominate_l0_category_name'], |
| 242 | + 'dominate_l1_category_id': agg_data['dominate_l1_category_id'], |
| 243 | + 'dominate_l1_category_name': agg_data['dominate_l1_category_name'], |
| 244 | + 'dominate_l1_category_asp': dom_l1_asp, |
| 245 | + }) |
| 246 | + |
| 247 | + #from IPython import embed; |
| 248 | + #embed() |
| 249 | + |
| 250 | + return data |
| 251 | + |
| 252 | + except ConnectionError as e: |
| 253 | + print(e) |
| 254 | + |
| 255 | + |
| 256 | +def average_asp(prices): |
| 257 | + total = sum(prices) |
| 258 | + if total > 0: |
| 259 | + return total / len(prices) |
| 260 | + else: |
| 261 | + return 0 |
| 262 | + |
| 263 | + |
| 264 | +def analyze_items(items, category_id, agg_data): |
| 265 | + |
| 266 | + for item in items: |
| 267 | + if not agg_data['cat_asp'].get(category_id, None): |
| 268 | + agg_data['cat_asp'][category_id] = [] |
| 269 | + |
| 270 | + agg_data['cat_asp'][category_id].append(float(item.sellingStatus.currentPrice.value)) |
| 271 | + if getattr(item.listingInfo, 'watchCount', None): |
| 272 | + agg_data['watch_count'] += int(item.listingInfo.watchCount) |
| 273 | + |
| 274 | + if getattr(item, 'postalCode', None): |
| 275 | + agg_data['postal_code'] = item.postalCode |
| 276 | + |
| 277 | + |
| 278 | +if __name__ == "__main__": |
| 279 | + print("Finding samples for SDK version %s" % ebaysdk.get_version()) |
| 280 | + (opts, args) = init_options() |
| 281 | + |
| 282 | + run(opts) |
0 commit comments