REDROOM

Navigation

Breadcrumbs

/ / python-scripts-home

Create

Create File | Create Folder | Upload Files | Run Command

Edit File Size: 10.65 KB

//python-scripts-home/merchant_upload.py

Text Base64

import requests
import traceback
import time
import logging
from datetime import datetime
from google.oauth2 import service_account
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from concurrent.futures import ThreadPoolExecutor, as_completed
from region_mappings.zip_to_region import ZIP_TO_REGION_ID
from region_mappings.city_to_region import CITY_TO_REGION_ID

# Configure logging
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s: %(message)s")

# === Google API Setup ===
SERVICE_ACCOUNT_FILE = './keys/quicklly-merchant-center.json'

SCOPES = ['https://www.googleapis.com/auth/content']
merchant_id = '5616033984'

credentials = service_account.Credentials.from_service_account_file(
    SERVICE_ACCOUNT_FILE, scopes=SCOPES
)
service = build('content', 'v2.1', credentials=credentials)

# === Retry logic ===
def retry_with_backoff(func, *args, **kwargs):
    max_retries = 5
    delay = 1
    for attempt in range(max_retries):
        try:
            return func(*args, **kwargs).execute()
        except HttpError as e:
            if e.resp.status in [429, 500, 503]:
                logging.warning(f"Retry {attempt+1}/{max_retries} after error {e.resp.status}. Waiting {delay}s...")
                time.sleep(delay)
                delay *= 2
            else:
                raise
    raise Exception("Max retries exceeded.")

'''
# === Helper: Region ID with fallback logic ===
def get_region_id(product):
    city = product.get('city', '').strip().lower()
    zipcode = product.get('zipcode', '').strip()
    
    # First try city mapping
    region_id = CITY_TO_REGION_ID.get(city)
    if region_id:
        return region_id
    
    # Fallback to zip code mapping
    if zipcode:
        region_id = ZIP_TO_REGION_ID.get(zipcode)
        if region_id:
            return region_id
    
    # Final fallback to unknown
    return "unknown"
'''

def get_region_id(product):
    city = product.get('city', '').strip().lower()
    zipcode = product.get('zipcode', '').strip()

region_id = CITY_TO_REGION_ID.get(city)
    if region_id:
        return region_id

region_id = ZIP_TO_REGION_ID.get(zipcode)
    if region_id:
        return region_id

# Debug log to investigate fallbacks
    logging.debug(f"[REGION MISSING] OfferID: {product.get('offerId')} | City: '{city}' | Zip: '{zipcode}'")

return "unknown"

# === Step 2: Fetch products ===
try:
    product_response = requests.post("https://devrestapi.goquicklly.com/productsForGoogle")
    product_response.raise_for_status()
    products = product_response.json().get("lstProducts", [])
except Exception as e:
    logging.error(f"Failed to fetch products: {e}")
    traceback.print_exc()
    products = []

# === Step 3: Upload products in chunks ===
def process_chunks(products):
    chunk_size = 750
    min_chunk_size = 100

while chunk_size >= min_chunk_size:
        for i in range(0, len(products), chunk_size):
            chunk = products[i:i+chunk_size]
            batch_entries = []

for idx, product in enumerate(chunk):
                offer_id = product.get('offerId')
                if not offer_id:
                    logging.warning("Skipped a product with missing offerId")
                    continue

action = product.get('action', 'insert').lower().strip()
                logging.info(f"Processing product: {offer_id} | Action: {action}")

link = product.get('link', '').strip()
                image_link = (product.get('imageLink', '').strip()
                              .replace('//', '/')
                              .replace('https:/', 'https://'))

description = product.get('description') or 'No description available'
                brand = product.get('brand') or 'Unbranded'
                gpc = product.get('googleProductCategory', '').strip()
                if not gpc or gpc.lower() == 'grocery':
                    gpc = 'Food, Beverages & Tobacco'

product_id = f'online:en:US:{offer_id}'

if action == 'insert':
                    try:
                        service.products().get(
                            merchantId=merchant_id,
                            productId=product_id
                        ).execute()
                        logging.info(f"Duplicate found for {offer_id}, switching to update")
                        action = 'update'
                    except HttpError as e:
                        if e.resp.status != 404:
                            raise

if action not in ['insert', 'update', 'delete']:
                    logging.warning(f"Skipped {offer_id}: invalid action '{action}'")
                    continue

if action == 'delete':
                    batch_entries.append({
                        'batchId': i + idx,
                        'merchantId': merchant_id,
                        'method': 'delete',
                        'productId': product_id
                    })
                else:
                    price = product.get('price')
                    if not isinstance(price, dict) or 'value' not in price:
                        price = {'value': '0.00', 'currency': 'USD'}

product_body = {
                        'offerId': offer_id,
                        'title': product.get('title', 'No Title'),
                        'description': description,
                        'link': link,
                        'imageLink': image_link,
                        'price': price,
                        'availability': product.get('availability', 'in stock'),
                        'condition': product.get('condition', 'new'),
                        'brand': brand,
                        'googleProductCategory': gpc,
                        'targetCountry': 'US',
                        'contentLanguage': 'en',
                        'channel': 'online',
                        'customLabel0': product.get('city', 'unknown'),
                        'customLabel1': product.get('state', ''),
                        'customLabel2': product.get('zipcode', ''),
                        'customLabel3': product.get('country', '')
                    }
                    batch_entries.append({
                        'batchId': i + idx,
                        'merchantId': merchant_id,
                        'method': action,
                        'product': product_body
                    })

if batch_entries:
                try:
                    batch_request = {'entries': batch_entries}
                    batch_response = retry_with_backoff(
                        service.products().custombatch, body=batch_request
                    )
                    for entry in batch_response.get('entries', []):
                        status = 'Success' if 'product' in entry else 'Failed'
                        logging.info(f"[{status}] Batch {entry['batchId']}: {entry.get('product', entry.get('errors'))}")
                except HttpError as e:
                    if e.resp.status == 413:
                        if chunk_size // 2 < min_chunk_size:
                            logging.error("Minimum chunk size reached. Cannot reduce further.")
                            return
                        logging.warning(f"Payload too large with chunk_size={chunk_size}. Reducing...")
                        chunk_size //= 2
                        break
                    else:
                        logging.error("Batch upload failed:", exc_info=e)
                        return
        else:
            break

# Process products
process_chunks(products)

# === Regional inventory batch functions ===
def send_regional_batch(batch_entries, batch_id):
    try:
        batch_request = {'entries': batch_entries}
        response = retry_with_backoff(
            service.regionalinventory().custombatch, body=batch_request
        )
        for entry in response.get('entries', []):
            status = 'Success' if 'regionalInventory' in entry else 'Failed'
            logging.info(f"[{status}] Regional Batch {batch_id}-{entry['batchId']}: {entry.get('regionalInventory', entry.get('errors'))}")
    except HttpError as e:
        logging.error(f"[Batch {batch_id}] Regional inventory batch update failed: {e}")

def batch_update_regional_inventory(products, batch_size=250, max_workers=4):
    all_batches = []
    current_batch = []
    batch_counter = 0
    skipped_products = 0

for idx, product in enumerate(products):
        offer_id = product.get('offerId')
        if not offer_id:
            skipped_products += 1
            continue

region_id = get_region_id(product)

# Debug: log region ID used for each product
        logging.debug(f"[REGION SUBMIT] OfferID: {offer_id} → RegionID: '{region_id}'")

if region_id == "unknown":
            skipped_products += 1
            continue

product_id = f'online:en:US:{offer_id}'
        price = product.get('price', {'value': '0.00', 'currency': 'USD'})
        availability = product.get('availability', 'in stock')

current_batch.append({
            'batchId': idx,
            'merchantId': merchant_id,
            'method': 'insert',
            'productId': product_id,
            'regionalInventory': {
                'regionId': region_id.strip(),
                'price': price,
                'availability': availability
            }
        })

if len(current_batch) >= batch_size:
            all_batches.append((current_batch[:], batch_counter))
            current_batch = []
            batch_counter += 1

if current_batch:
        all_batches.append((current_batch, batch_counter))

logging.info(f"Regional inventory update: Processing {len(products) - skipped_products} products in {len(all_batches)} batches")
    if skipped_products > 0:
        logging.info(f"Skipped {skipped_products} products due to missing offerIds or regionId")

with ThreadPoolExecutor(max_workers=max_workers) as executor:
        futures = [executor.submit(send_regional_batch, batch, bid) for batch, bid in all_batches]
        for future in as_completed(futures):
            future.result()

# === Step 4: Regional inventory push (BATCHED) ===
batch_update_regional_inventory(products)

# === Step 5: Update crawl status ===
for product in products:
    offer_id = product.get('offerId')
    if not offer_id:
        continue
    try:
        crawl_response = requests.post(
            "https://devrestapi.goquicklly.com/updateProductCrowlStatus",
            json={"offerid": offer_id}
        )
        logging.info(f'Updated crawl status for {offer_id}: {crawl_response.json()}')
    except Exception as e:
        logging.error(f'Failed to update crawl status for {offer_id}: {e}')

Exit & Reset Text mode: syntax highlighting auto-detects file type.

Directory Contents

Dirs: 4 Ã— Files: 4

Sort:

Enable drag-to-move

Name	Size	Perms	Modified	Actions
keys DIR	-	drwxr-xr-x	2025-07-22 17:20:04	Edit Download recursive recursive
logs DIR	-	drwxr-xr-x	2025-07-22 18:23:22	Edit Download recursive recursive
region_mappings DIR	-	drwxr-xr-x	2025-07-31 17:00:29	Edit Download recursive recursive
tools DIR	-	drwxr-xr-x	2025-07-31 17:01:47	Edit Download recursive recursive
merchant_upload.py	10.65 KB	lrw-r--r--	2025-07-31 17:03:46	Edit Download recursive recursive
merchant_upload.sh	127 B	lrwxrwxrwx	2025-07-23 16:56:47	Edit Download recursive recursive
README.md	2.86 KB	lrw-r--r--	2025-07-22 17:19:18	Edit Download recursive recursive
requirements.txt	46 B	lrw-r--r--	2025-07-22 17:19:18	Edit Download recursive recursive