Depending on the type of website you run, sometimes the default widgets are useless to your purpose so rather than have them load everytime you access the widgets page, you can simply add this to your functions.php file. Obviously you can remove any from this list to keep them.
function remove_default_wp_widgets() {
unregister_widget('WP_Widget_Pages');
unregister_widget('WP_Widget_Calendar');
unregister_widget('WP_Widget_Archives');
unregister_widget('WP_Widget_Links');
unregister_widget('WP_Widget_Meta');
unregister_widget('WP_Widget_Search');
unregister_widget('WP_Widget_Text');
unregister_widget('WP_Widget_Categories');
unregister_widget('WP_Widget_Recent_Posts');
unregister_widget('WP_Widget_Recent_Comments');
unregister_widget('WP_Widget_RSS');
unregister_widget('WP_Widget_Tag_Cloud');
}
add_action('widgets_init', 'remove_default_wp_widgets', 1);