Totally possible, we do it on WordPress.com
Easiest approach is to setup a new virtual host for that subdomain before your WPMU virtualhost in Apache. You can then catch it before it goes to the WPMU install which usually catches all subdomains.
You can also do it from the same virtualhost for a thrill. That does require some .htaccess hackery. This is how WordPress.com is done.
NB: these rules are for bbPress 0.9 and are supposed to catch all our localised forum domains – it also may not work for everyone. The important part of the rules to note is the skip (S=30) rule at the start.
# Skip forum rules if not in *.forums.wordpress.com
RewriteCond %{HTTP_HOST} !^(.+.)*forums.wordpress.com
RewriteRule .* -
# Forum rules
# If you change the total number of rules, the skip rule above must be updated too
RewriteRule ^/?$ /bbpress/ [L]
RewriteRule ^bb-admin$ /bb-admin/ [R,L]
RewriteRule ^bb-admin/(.*)$ /bbpress/bb-admin/$1 [L,QSA]
RewriteRule ^bb-images/(.*)$ /bbpress/bb-images/$1 [L,QSA]
RewriteRule ^bb-includes/js/(.*)$ /bbpress/bb-includes/js/$1 [L,QSA]
RewriteRule ^([^/]+).php(.*)$ /bbpress/$1.php$2 [L,QSA]
RewriteRule ^forum/([^/]+)/page/([0-9]+)/?$ /bbpress/forum.php?id=$1&page=$2 [L,QSA]
RewriteRule ^forum/([^/]+)/?$ /bbpress/forum.php?id=$1 [L,QSA]
RewriteRule ^forum/?$ /bbpress/ [R,L]
RewriteRule ^topic/([^/]+)/page/([0-9]+)/?$ /bbpress/topic.php?id=$1&page=$2 [L,QSA]
RewriteRule ^topic/([^/]+)/?$ /bbpress/topic.php?id=$1 [L,QSA]
RewriteRule ^topic/?$ /bbpress/ [R,L]
RewriteRule ^tags/([^/]+)/page/([0-9]+)/?$ /bbpress/tags.php?tag=$1&page=$2 [L,QSA]
RewriteRule ^tags/([^/]+)/?$ /bbpress/tags.php?tag=$1 [L,QSA]
RewriteRule ^tags/?$ /bbpress/tags.php [L,QSA]
RewriteRule ^profile/([^/]+)/page/([0-9]+)/?$ /bbpress/profile.php?id=$1&page=$2 [L,QSA]
RewriteRule ^profile/([^/]+)/([^/]+)/?$ /bbpress/profile.php?id=$1&tab=$2 [L,QSA]
RewriteRule ^profile/([^/]+)/([^/]+)/page/([0-9]+)/?$ /bbpress/profile.php?id=$1&tab=$2&page=$3 [L,QSA]
RewriteRule ^profile/([^/]+)/?$ /bbpress/profile.php?id=$1 [L,QSA]
RewriteRule ^profile/?$ /bbpress/profile.php [L,QSA]
RewriteRule ^view/([^/]+)/page/([0-9]+)/?$ /bbpress/view.php?view=$1&page=$2 [L,QSA]
RewriteRule ^view/([^/]+)/?$ /bbpress/view.php?view=$1 [L,QSA]
RewriteRule ^rss/?$ /bbpress/rss.php [L,QSA]
RewriteRule ^rss/topics/?$ /bbpress/rss.php?topics=1 [L,QSA]
RewriteRule ^rss/forum/([^/]+)/?$ /bbpress/rss.php?forum=$1 [L,QSA]
RewriteRule ^rss/forum/([^/]+)/topics/?$ /bbpress/rss.php?forum=$1&topics=1 [L,QSA]
RewriteRule ^rss/topic/([^/]+)/?$ /bbpress/rss.php?topic=$1 [L,QSA]
RewriteRule ^rss/tags/([^/]+)/?$ /bbpress/rss.php?tag=$1 [L,QSA]
RewriteRule ^rss/profile/([^/]+)/?$ /bbpress/rss.php?profile=$1 [L,QSA]
RewriteRule ^rss/view/([^/]+)/?$ /bbpress/rss.php?view=$1 [L,QSA]
# If you add more forum rules then you must update the skip rule above them as well
