[11430] in www-talk@info.cern.ch
Re: user agent label format
daemon@ATHENA.MIT.EDU (Andrew Prendergast)
Tue Sep 14 15:28:27 1999
From: "Andrew Prendergast" <ap@geekz.com.au>
To: "Koen Holtman" <koen@win.tue.nl>, "Dan Connolly" <connolly@w3.org>
Cc: <dberansky@ucsd.edu>, <www-talk@w3.org>
Date: Wed, 15 Sep 1999 05:18:36 +1000
Resent-From: www-talk@w3.org
This is a multi-part message in MIME format.
------=_NextPart_000_0404_01BEFF39.C30D4C80
Content-Type: text/plain;
charset="iso-8859-1"
Content-Transfer-Encoding: 7bit
Based on many years of writing all sorts of convoluted weirdo internet based
stuff, I have come up with the attached C solution to the problem... be
warned, it is not 100% - but it works in most cases.
I basically use this to process the useragent, then assign each unique
string an ID and store it in a database for later use in apps ranging from
web stats to serving different content to different browsers.
If you want to test it - rip out a few thousand lines from a useragent log,
then pass them through the attached function. You should then get the idea.
Regards,
ap.
----- Original Message -----
From: Koen Holtman <koen@win.tue.nl>
To: Dan Connolly <connolly@w3.org>
Cc: <dberansky@ucsd.edu>; <www-talk@w3.org>
Sent: Wednesday, September 15, 1999 2:27 AM
Subject: Re: user agent label format
> Dan Connolly:
> >
> >Dmitry Beransky wrote:
> >>
> >> It seems that despite the fact that most agents (at least the ones I
pulled
> >> off my server logs) follow the format specified in the HTTP spec, it's
> >> still impossible to determine with 100% accuracy the agent's make,
model,
> >> os platform, etc., since most of this info is stored in an unstructured
> >> comment field. Is there any work done on trying to further standardize
the
> >> agent data?
> >
> >Not that I'm aware of.
>
> Some additional observations: the HTTP spec implies that the
> User-Agent field is free-form, and in at least one browser (Lynx), the
> end users can set this field to anything they want.
>
> This implies to me that developing a standard format for the User-Agent
> field is a lost cause. The best hope with respect to reliable browser
> identification is the wide deployment of one of the content negotiation
> mechanisms referenced elsewhere in this thread.
>
> >Dan Connolly, W3C
>
> Koen.
>
------=_NextPart_000_0404_01BEFF39.C30D4C80
Content-Type: text/plain;
name="uagent_fix.txt"
Content-Transfer-Encoding: quoted-printable
Content-Disposition: attachment;
filename="uagent_fix.txt"
int uagent_fix(char *buffer, int maxlen) {
/* makes useragent strings into something=20
* clean and simple ap@geekz.com.au */
int x, y=3D0, len;
char junk[128];
/* convert to lowercase */
strnlower(buffer, maxlen);
/* trim any leading space off */
len =3D strlen(buffer);
for(x =3D 0; x < len && x < maxlen; x++)
if( !isspace(buffer[x]) )
break;
if(x > 0)
strncpy(buffer, &buffer[x], maxlen);
/* delete any weird characters & duplicate spaces */
len =3D strlen(buffer);
for ( x =3D 0; x < len && x < maxlen && buffer[x] !=3D 0; x++ )
if ( ( isspace(buffer[x]) && isspace(buffer[x + 1]) ) || =
(!isprint(buffer[x]) && !isspace(buffer[x])) )
{
strncpy(&buffer[x], &buffer[x + 1], maxlen - x);
x--;
}
/* change mozilla to netscape */
if( fastcmp(buffer, "mozilla/", 8) ) {
strncpy(junk, buffer, sizeof(junk) - 1);
strncpy(junk, &junk[8], sizeof(junk) - 1);
strncpy(buffer, "netscape ", maxlen);
strncpy(&buffer[9], junk, maxlen - 9);
}
/* change microsoft internet explorer to msie */
if( fastcmp(buffer, "microsoft internet explorer", 27) ) {
strncpy(junk, buffer, sizeof(junk) - 1);
strncpy(junk, &junk[27], sizeof(junk) - 1);
strncpy(buffer, "msie", maxlen);
strncpy(&buffer[4], junk, maxlen - 9);
}
/* detect comaptibility hack */
len=3Dstrlen(buffer);
for( x =3D 0; x < len && x < maxlen; x++ )
if( buffer[x] =3D=3D '(' && fastcmp(&buffer[x], "(compatible;", 12) )
goto chack;
goto nochack; /* jump over compatibility hack */
/* we only get here if we have a compatibility hack on our hands */
chack:
x +=3D 13;
for( ; x < len && x < maxlen; x++ )
{
if ( buffer[x] =3D=3D ';' || buffer[x] =3D=3D ')' )
break;
buffer[y] =3D buffer[x];
y++;
}
buffer[y] =3D 0;
len =3D strlen(buffer);
/* check if a mozilla/... was on the end of the compat string (some =
dumb programmer fucked up) */
for( x =3D 0; x < len - 8 && x < maxlen - 7; x++ )
if ( buffer[x] =3D=3D 'm' &&
buffer[x + 1] =3D=3D 'o' &&
buffer[x + 2] =3D=3D 'z' &&
buffer[x + 3] =3D=3D 'i' &&
buffer[x + 4] =3D=3D 'l' &&
buffer[x + 5] =3D=3D 'l' &&
buffer[x + 6] =3D=3D 'a' &&
buffer[x + 7] =3D=3D '/' ) buffer[x] =3D 0;
/* some msie's come out as msie+4 or msie.4 - change to msie/... */
if ( buffer[0] =3D=3D 'm' && buffer[1] =3D=3D 's' &&
buffer[2] =3D=3D 'i' && buffer[3] =3D=3D 'e' &&
buffer[4] !=3D '/' && !isalnum(buffer[4]) ) buffer[4] =3D '/';
nochack:
/* trim trailing junk in brackets off */
for( x =3D 0; x < len && x < maxlen; x++ )
if( buffer[x] =3D=3D '(' )
buffer[x] =3D 0;
/* trim trailing spaces off */
len =3D strlen(buffer);
if( maxlen < len ) len =3D maxlen;
for( x =3D len - 1; x >=3D 1; x-- )
if ( isspace(buffer[x]) )
buffer[x] =3D 0;
else break;
/* if msie/ or netscape/ */
if ( strncmp(buffer, "msie", 4) =3D=3D 0 ||
strncmp(buffer, "netscape", 8) =3D=3D 0)
{
/* remove any crap after the version # */
len =3D strlen(buffer);
for( y =3D x =3D 0; x < len && x < maxlen; x++ )
if ( buffer[x] =3D=3D ' ' )
{
if ( y++ )
{
buffer[x] =3D 0;
break;
}
else buffer[x] =3D '/';
}
/* remove -c... c-... -... and trailing c crap */
if ( strncmp(buffer, "netscape", 8) =3D=3D 0 )
{
len =3D strlen(buffer);
for ( x =3D 0; x < len && x < maxlen; x++ )
if ( (buffer[x] =3D=3D '-' && buffer[x + 1] =3D=3D 'c') ||
(buffer[x] =3D=3D 'c' && buffer[x + 1] =3D=3D '-') ||
(buffer[x] =3D=3D '-') )
{
buffer[x] =3D 0; break;
}
len =3D strlen(buffer);
if ( buffer[x - 1] =3D=3D 'c' ) buffer[x - 1] =3D 0;
}
}
/* delete any weird characters and convert _'s to spaces */
len =3D strlen(buffer);
for ( x =3D 0; x < len && x < maxlen && buffer[x] !=3D 0; x++ )
{
if ( buffer[x] >=3D 'a' && buffer[x] <=3D 'z' ) /* do nothing */;
else if ( buffer[x] >=3D '0' && buffer[x] <=3D '9' ) /* do nothing */;
else if ( buffer[x] =3D=3D '.' || buffer[x] =3D=3D '/' ||
buffer[x] =3D=3D '-' || buffer[x] =3D=3D '[' ||
buffer[x] =3D=3D ']' || buffer[x] =3D=3D ' ') /* do nothing */;
else if ( buffer[x] =3D=3D '_' ) buffer[x] =3D ' ';
else
{
strncpy(&buffer[x], &buffer[x + 1], maxlen - x);
x--;
}
}
/* trim any leading space off result */
len =3D strlen(buffer);
for( x =3D 0; x < len && x < maxlen; x++ )
if( !isspace(buffer[x]) ) break;
if( x > 0 )
strncpy(buffer, &buffer[x], maxlen);
/* anything 1 or 2 chars is considered 'unknown' */
len =3D strlen(buffer);
if ( len <=3D 2 ) strncpy(buffer, "unknown", maxlen);
return 1;
}
------=_NextPart_000_0404_01BEFF39.C30D4C80--
