Preview: merchant_upload.py
Size: 10.65 KB
//python-scripts-home/merchant_upload.py
import requests
import traceback
import time
import logging
from datetime import datetime
from google.oauth2 import service_account
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from concurrent.futures import ThreadPoolExecutor, as_completed
from region_mappings.zip_to_region import ZIP_TO_REGION_ID
from region_mappings.city_to_region import CITY_TO_REGION_ID
# Configure logging
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s: %(message)s")
# === Google API Setup ===
SERVICE_ACCOUNT_FILE = './keys/quicklly-merchant-center.json'
SCOPES = ['https://www.googleapis.com/auth/content']
merchant_id = '5616033984'
credentials = service_account.Credentials.from_service_account_file(
SERVICE_ACCOUNT_FILE, scopes=SCOPES
)
service = build('content', 'v2.1', credentials=credentials)
# === Retry logic ===
def retry_with_backoff(func, *args, **kwargs):
max_retries = 5
delay = 1
for attempt in range(max_retries):
try:
return func(*args, **kwargs).execute()
except HttpError as e:
if e.resp.status in [429, 500, 503]:
logging.warning(f"Retry {attempt+1}/{max_retries} after error {e.resp.status}. Waiting {delay}s...")
time.sleep(delay)
delay *= 2
else:
raise
raise Exception("Max retries exceeded.")
'''
# === Helper: Region ID with fallback logic ===
def get_region_id(product):
city = product.get('city', '').strip().lower()
zipcode = product.get('zipcode', '').strip()
# First try city mapping
region_id = CITY_TO_REGION_ID.get(city)
if region_id:
return region_id
# Fallback to zip code mapping
if zipcode:
region_id = ZIP_TO_REGION_ID.get(zipcode)
if region_id:
return region_id
# Final fallback to unknown
return "unknown"
'''
def get_region_id(product):
city = product.get('city', '').strip().lower()
zipcode = product.get('zipcode', '').strip()
region_id = CITY_TO_REGION_ID.get(city)
if region_id:
return region_id
region_id = ZIP_TO_REGION_ID.get(zipcode)
if region_id:
return region_id
# Debug log to investigate fallbacks
logging.debug(f"[REGION MISSING] OfferID: {product.get('offerId')} | City: '{city}' | Zip: '{zipcode}'")
return "unknown"
# === Step 2: Fetch products ===
try:
product_response = requests.post("https://devrestapi.goquicklly.com/productsForGoogle")
product_response.raise_for_status()
products = product_response.json().get("lstProducts", [])
except Exception as e:
logging.error(f"Failed to fetch products: {e}")
traceback.print_exc()
products = []
# === Step 3: Upload products in chunks ===
def process_chunks(products):
chunk_size = 750
min_chunk_size = 100
while chunk_size >= min_chunk_size:
for i in range(0, len(products), chunk_size):
chunk = products[i:i+chunk_size]
batch_entries = []
for idx, product in enumerate(chunk):
offer_id = product.get('offerId')
if not offer_id:
logging.warning("Skipped a product with missing offerId")
continue
action = product.get('action', 'insert').lower().strip()
logging.info(f"Processing product: {offer_id} | Action: {action}")
link = product.get('link', '').strip()
image_link = (product.get('imageLink', '').strip()
.replace('//', '/')
.replace('https:/', 'https://'))
description = product.get('description') or 'No description available'
brand = product.get('brand') or 'Unbranded'
gpc = product.get('googleProductCategory', '').strip()
if not gpc or gpc.lower() == 'grocery':
gpc = 'Food, Beverages & Tobacco'
product_id = f'online:en:US:{offer_id}'
if action == 'insert':
try:
service.products().get(
merchantId=merchant_id,
productId=product_id
).execute()
logging.info(f"Duplicate found for {offer_id}, switching to update")
action = 'update'
except HttpError as e:
if e.resp.status != 404:
raise
if action not in ['insert', 'update', 'delete']:
logging.warning(f"Skipped {offer_id}: invalid action '{action}'")
continue
if action == 'delete':
batch_entries.append({
'batchId': i + idx,
'merchantId': merchant_id,
'method': 'delete',
'productId': product_id
})
else:
price = product.get('price')
if not isinstance(price, dict) or 'value' not in price:
price = {'value': '0.00', 'currency': 'USD'}
product_body = {
'offerId': offer_id,
'title': product.get('title', 'No Title'),
'description': description,
'link': link,
'imageLink': image_link,
'price': price,
'availability': product.get('availability', 'in stock'),
'condition': product.get('condition', 'new'),
'brand': brand,
'googleProductCategory': gpc,
'targetCountry': 'US',
'contentLanguage': 'en',
'channel': 'online',
'customLabel0': product.get('city', 'unknown'),
'customLabel1': product.get('state', ''),
'customLabel2': product.get('zipcode', ''),
'customLabel3': product.get('country', '')
}
batch_entries.append({
'batchId': i + idx,
'merchantId': merchant_id,
'method': action,
'product': product_body
})
if batch_entries:
try:
batch_request = {'entries': batch_entries}
batch_response = retry_with_backoff(
service.products().custombatch, body=batch_request
)
for entry in batch_response.get('entries', []):
status = 'Success' if 'product' in entry else 'Failed'
logging.info(f"[{status}] Batch {entry['batchId']}: {entry.get('product', entry.get('errors'))}")
except HttpError as e:
if e.resp.status == 413:
if chunk_size // 2 < min_chunk_size:
logging.error("Minimum chunk size reached. Cannot reduce further.")
return
logging.warning(f"Payload too large with chunk_size={chunk_size}. Reducing...")
chunk_size //= 2
break
else:
logging.error("Batch upload failed:", exc_info=e)
return
else:
break
# Process products
process_chunks(products)
# === Regional inventory batch functions ===
def send_regional_batch(batch_entries, batch_id):
try:
batch_request = {'entries': batch_entries}
response = retry_with_backoff(
service.regionalinventory().custombatch, body=batch_request
)
for entry in response.get('entries', []):
status = 'Success' if 'regionalInventory' in entry else 'Failed'
logging.info(f"[{status}] Regional Batch {batch_id}-{entry['batchId']}: {entry.get('regionalInventory', entry.get('errors'))}")
except HttpError as e:
logging.error(f"[Batch {batch_id}] Regional inventory batch update failed: {e}")
def batch_update_regional_inventory(products, batch_size=250, max_workers=4):
all_batches = []
current_batch = []
batch_counter = 0
skipped_products = 0
for idx, product in enumerate(products):
offer_id = product.get('offerId')
if not offer_id:
skipped_products += 1
continue
region_id = get_region_id(product)
# Debug: log region ID used for each product
logging.debug(f"[REGION SUBMIT] OfferID: {offer_id} → RegionID: '{region_id}'")
if region_id == "unknown":
skipped_products += 1
continue
product_id = f'online:en:US:{offer_id}'
price = product.get('price', {'value': '0.00', 'currency': 'USD'})
availability = product.get('availability', 'in stock')
current_batch.append({
'batchId': idx,
'merchantId': merchant_id,
'method': 'insert',
'productId': product_id,
'regionalInventory': {
'regionId': region_id.strip(),
'price': price,
'availability': availability
}
})
if len(current_batch) >= batch_size:
all_batches.append((current_batch[:], batch_counter))
current_batch = []
batch_counter += 1
if current_batch:
all_batches.append((current_batch, batch_counter))
logging.info(f"Regional inventory update: Processing {len(products) - skipped_products} products in {len(all_batches)} batches")
if skipped_products > 0:
logging.info(f"Skipped {skipped_products} products due to missing offerIds or regionId")
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [executor.submit(send_regional_batch, batch, bid) for batch, bid in all_batches]
for future in as_completed(futures):
future.result()
# === Step 4: Regional inventory push (BATCHED) ===
batch_update_regional_inventory(products)
# === Step 5: Update crawl status ===
for product in products:
offer_id = product.get('offerId')
if not offer_id:
continue
try:
crawl_response = requests.post(
"https://devrestapi.goquicklly.com/updateProductCrowlStatus",
json={"offerid": offer_id}
)
logging.info(f'Updated crawl status for {offer_id}: {crawl_response.json()}')
except Exception as e:
logging.error(f'Failed to update crawl status for {offer_id}: {e}')
Directory Contents
Dirs: 4 × Files: 4